alertmanager: configure receiver secrets
This commit is contained in:
parent
00ce1a8a26
commit
6ee8d3372a
File diff suppressed because it is too large
Load Diff
@ -8,6 +8,8 @@ alertmanager_web_external_url: https://monitor.kill0.net/alertmanager
|
|||||||
prometheus_web_route_prefix: /
|
prometheus_web_route_prefix: /
|
||||||
alertmanager_web_route_prefix: /
|
alertmanager_web_route_prefix: /
|
||||||
|
|
||||||
|
prometheus_file_sd_config_d_files: []
|
||||||
|
|
||||||
prometheus_config:
|
prometheus_config:
|
||||||
global:
|
global:
|
||||||
scrape_interval: 15s
|
scrape_interval: 15s
|
||||||
@ -16,6 +18,10 @@ prometheus_config:
|
|||||||
region: dallas
|
region: dallas
|
||||||
provider: linode
|
provider: linode
|
||||||
replica: A
|
replica: A
|
||||||
|
remote_write:
|
||||||
|
- url: http://localhost:9009/api/v1/push
|
||||||
|
headers:
|
||||||
|
X-Scope-OrgID: kill0-net
|
||||||
alerting:
|
alerting:
|
||||||
alertmanagers:
|
alertmanagers:
|
||||||
- static_configs:
|
- static_configs:
|
||||||
@ -177,6 +183,77 @@ prometheus_config:
|
|||||||
static_configs:
|
static_configs:
|
||||||
- targets:
|
- targets:
|
||||||
- "localhost:3002"
|
- "localhost:3002"
|
||||||
|
# - job_name: process-exporter
|
||||||
|
# scrape_interval: 5s
|
||||||
|
# static_configs:
|
||||||
|
# - targets:
|
||||||
|
# - "localhost:9256"
|
||||||
|
- job_name: loki
|
||||||
|
scrape_interval: 5s
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- "localhost:3100"
|
||||||
|
- job_name: promtail
|
||||||
|
scrape_interval: 5s
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- jump0.kill0.net:9080
|
||||||
|
- mine0.kill0.net:9080
|
||||||
|
- job_name: gitea
|
||||||
|
scrape_interval: 5s
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost:3001
|
||||||
|
- job_name: karma
|
||||||
|
scrape_interval: 5s
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost:8080
|
||||||
|
- job_name: kthxbye
|
||||||
|
scrape_interval: 5s
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost:8081
|
||||||
|
- job_name: smokeping
|
||||||
|
scrape_interval: 5s
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost:9374
|
||||||
|
- job_name: mimir
|
||||||
|
scrape_interval: 5s
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost:9009
|
||||||
|
- &snmp_job
|
||||||
|
job_name: snmp
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 172.16.100.1
|
||||||
|
- 172.16.100.2
|
||||||
|
metrics_path: /snmp
|
||||||
|
params:
|
||||||
|
auth: [public_v2]
|
||||||
|
module:
|
||||||
|
- if_mib
|
||||||
|
- ip_mib
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__address__]
|
||||||
|
target_label: __param_target
|
||||||
|
- source_labels: [__param_target]
|
||||||
|
target_label: instance
|
||||||
|
- target_label: __address__
|
||||||
|
replacement: 127.0.0.1:9116
|
||||||
|
- job_name: snmp_exporter
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost:9116
|
||||||
|
- <<: *snmp_job
|
||||||
|
job_name: snmp-long
|
||||||
|
scrape_interval: 30s
|
||||||
|
scrape_timeout: 30s
|
||||||
|
static_configs:
|
||||||
|
- targets: []
|
||||||
|
|
||||||
rule_files:
|
rule_files:
|
||||||
- rules.yaml
|
- rules.yaml
|
||||||
|
|
||||||
@ -228,6 +305,10 @@ prometheus_rules_config:
|
|||||||
expr: up{job=~"thanos.+"} == 0
|
expr: up{job=~"thanos.+"} == 0
|
||||||
labels:
|
labels:
|
||||||
severity: critical
|
severity: critical
|
||||||
|
- alert: Down
|
||||||
|
expr: up == 0
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
- alert: FileSystemUsage
|
- alert: FileSystemUsage
|
||||||
expr: ((node_filesystem_size_bytes{mountpoint!~"fuse.lxcfs|tmpfs"} - node_filesystem_free_bytes) / node_filesystem_size_bytes) > 0.80
|
expr: ((node_filesystem_size_bytes{mountpoint!~"fuse.lxcfs|tmpfs"} - node_filesystem_free_bytes) / node_filesystem_size_bytes) > 0.80
|
||||||
for: 1m
|
for: 1m
|
||||||
@ -280,6 +361,13 @@ prometheus_rules_config:
|
|||||||
# summary: Certificates expiring in < 14 days
|
# summary: Certificates expiring in < 14 days
|
||||||
summary: "{% raw %}Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}){% endraw %}"
|
summary: "{% raw %}Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}){% endraw %}"
|
||||||
description: "{% raw %}SSL certificate expires in 14 days\n VALUE = {{ $value }}\n LABELS = {{ $labels }}{% endraw %}"
|
description: "{% raw %}SSL certificate expires in 14 days\n VALUE = {{ $value }}\n LABELS = {{ $labels }}{% endraw %}"
|
||||||
|
- name: snmp.rules
|
||||||
|
rules:
|
||||||
|
- alert: PortDown
|
||||||
|
expr: ifAdminStatus{ifName=~"(Gi|eth).+", ifAlias!~".+laptop|notebook.+"} == 1 and ifOperStatus == 2
|
||||||
|
for: 1m
|
||||||
|
- alert: PortFlapping
|
||||||
|
expr: changes(ifOperStatus{ifName=~"(Gi|eth).+"}[5m]) > 2
|
||||||
|
|
||||||
blackbox_exporter_config:
|
blackbox_exporter_config:
|
||||||
modules:
|
modules:
|
||||||
@ -309,34 +397,6 @@ blackbox_exporter_config:
|
|||||||
http:
|
http:
|
||||||
method: GET
|
method: GET
|
||||||
|
|
||||||
# route:
|
|
||||||
# receiver: pushover-receiver
|
|
||||||
# mute_time_intervals:
|
|
||||||
# - quiet_hours
|
|
||||||
# routes:
|
|
||||||
# - receiver: blackhole
|
|
||||||
# match:
|
|
||||||
# alertname: MaintenanceMode
|
|
||||||
# #- receiver: blackhole
|
|
||||||
# # match:
|
|
||||||
# # alertname: QuietHours
|
|
||||||
# receivers:
|
|
||||||
# - name: blackhole
|
|
||||||
# - name: pushover-receiver
|
|
||||||
# pushover_configs:
|
|
||||||
# - token: "{{ vault_pushover_token }}"
|
|
||||||
# user_key: "{{ vault_pushover_user_key }}"
|
|
||||||
# inhibit_rules:
|
|
||||||
# - source_match:
|
|
||||||
# alertname: MaintenanceMode
|
|
||||||
# #- source_match:
|
|
||||||
# # alertname: QuietHours
|
|
||||||
# time_intervals:
|
|
||||||
# - name: quiet_hours
|
|
||||||
# times:
|
|
||||||
# - start_time: 03:00
|
|
||||||
# end_time: 15:00
|
|
||||||
|
|
||||||
alertmanager_config:
|
alertmanager_config:
|
||||||
inhibit_rules:
|
inhibit_rules:
|
||||||
- source_match:
|
- source_match:
|
||||||
@ -345,9 +405,13 @@ alertmanager_config:
|
|||||||
- name: blackhole
|
- name: blackhole
|
||||||
- name: pushover-receiver
|
- name: pushover-receiver
|
||||||
pushover_configs:
|
pushover_configs:
|
||||||
- token: agwd6wv7xveakykb8e5rz7rw3eg2v3
|
- token: "{{ vault_alertmanager_pushover_token }}"
|
||||||
user_key: 28G1x3lT4oUtlck50R1H3e6j8kDHjb
|
user_key: 28G1x3lT4oUtlck50R1H3e6j8kDHjb
|
||||||
|
- name: discord
|
||||||
|
discord_configs:
|
||||||
|
- webhook_url: "{{ vault_alertmanager_discord_webhook_url }}"
|
||||||
route:
|
route:
|
||||||
|
repeat_interval: 24h
|
||||||
receiver: pushover-receiver
|
receiver: pushover-receiver
|
||||||
routes:
|
routes:
|
||||||
- match:
|
- match:
|
||||||
@ -359,6 +423,8 @@ alertmanager_config:
|
|||||||
- receiver: pushover-receiver
|
- receiver: pushover-receiver
|
||||||
mute_time_intervals:
|
mute_time_intervals:
|
||||||
- quiet_hours
|
- quiet_hours
|
||||||
|
continue: true
|
||||||
|
- receiver: discord
|
||||||
time_intervals:
|
time_intervals:
|
||||||
- name: quiet_hours
|
- name: quiet_hours
|
||||||
time_intervals:
|
time_intervals:
|
||||||
@ -419,3 +485,25 @@ karma_config:
|
|||||||
thanos_bucket_config: "{{ vault_thanos_bucket_config }}"
|
thanos_bucket_config: "{{ vault_thanos_bucket_config }}"
|
||||||
|
|
||||||
kthxbye_listen: :8081
|
kthxbye_listen: :8081
|
||||||
|
|
||||||
|
smokeping_prober_config:
|
||||||
|
targets:
|
||||||
|
- hosts:
|
||||||
|
- dns.google
|
||||||
|
- vpn-home.kill0.net
|
||||||
|
- ping-home.kill0.net
|
||||||
|
- vpn1-sch.corp.nmi.com
|
||||||
|
- gp-chi.ops.nmi.com
|
||||||
|
- gp-ash.ops.nmi.com
|
||||||
|
- 169.254.0.2
|
||||||
|
- 172.16.100.1
|
||||||
|
- 172.16.100.2
|
||||||
|
- 172.16.10.16
|
||||||
|
network: ip4
|
||||||
|
- hosts:
|
||||||
|
- dns.google
|
||||||
|
- ping-home.kill0.net
|
||||||
|
- fc00::ffff:169.255.0.2
|
||||||
|
- fc00::ffff:169.255.0.16
|
||||||
|
network: ip6
|
||||||
|
|
||||||
|
|||||||
@ -39,8 +39,11 @@ wireguard_interfaces:
|
|||||||
|
|
||||||
restic_tidy_enabled: true
|
restic_tidy_enabled: true
|
||||||
|
|
||||||
|
nginx_htpasswd_files: "{{ vault_nginx_htpasswd_files }}"
|
||||||
|
|
||||||
nginx_vhosts:
|
nginx_vhosts:
|
||||||
cavicc:
|
cavicc:
|
||||||
|
server:
|
||||||
- server_name: cavi.cc
|
- server_name: cavi.cc
|
||||||
root: /var/www/cavicc
|
root: /var/www/cavicc
|
||||||
listen:
|
listen:
|
||||||
@ -53,7 +56,66 @@ nginx_vhosts:
|
|||||||
- server_name: cavi.cc
|
- server_name: cavi.cc
|
||||||
root: /var/www/cavicc
|
root: /var/www/cavicc
|
||||||
listen:
|
listen:
|
||||||
- 443 ssl http2
|
- 443 ssl
|
||||||
- "[::]:443 ssl http2"
|
- "[::]:443 ssl"
|
||||||
ssl_certificate: /etc/letsencrypt/live/cavi.cc/fullchain.pem
|
ssl_certificate: /var/lib/lego/certificates/cavi.cc.crt
|
||||||
ssl_certificate_key: /etc/letsencrypt/live/cavi.cc/privkey.pem
|
ssl_certificate_key: /var/lib/lego/certificates/cavi.cc.key
|
||||||
|
# ssl_certificate: /etc/letsencrypt/live/cavi.cc/fullchain.pem
|
||||||
|
# ssl_certificate_key: /etc/letsencrypt/live/cavi.cc/privkey.pem
|
||||||
|
raw: |
|
||||||
|
location / {
|
||||||
|
add_header Alt-Svc 'h3=":$server_port"; ma=86400';
|
||||||
|
}
|
||||||
|
|
||||||
|
proxy:
|
||||||
|
upstream:
|
||||||
|
- name: loki_backend
|
||||||
|
server:
|
||||||
|
- localhost:3100
|
||||||
|
#- name: prometheus_backend
|
||||||
|
# server:
|
||||||
|
# - localhost:9090
|
||||||
|
map:
|
||||||
|
- name: $http_upgrade
|
||||||
|
variable: $connection_upgrade
|
||||||
|
content:
|
||||||
|
default: upgrade
|
||||||
|
'': close
|
||||||
|
server:
|
||||||
|
- server_name: proxy.kill0.net
|
||||||
|
root: /var/empty
|
||||||
|
listen:
|
||||||
|
- 80
|
||||||
|
- "[::]:80"
|
||||||
|
raw: |
|
||||||
|
location / {
|
||||||
|
return 301 https://$server_name$request_uri;
|
||||||
|
}
|
||||||
|
- server_name: proxy.kill0.net
|
||||||
|
root: /var/empty
|
||||||
|
listen:
|
||||||
|
- 443 ssl
|
||||||
|
- "[::]:443 ssl"
|
||||||
|
# ssl_certificate: /etc/letsencrypt/live/proxy.kill0.net/fullchain.pem
|
||||||
|
# ssl_certificate_key: /etc/letsencrypt/live/proxy.kill0.net/privkey.pem
|
||||||
|
ssl_certificate: /var/lib/lego/certificates/proxy.kill0.net.crt
|
||||||
|
ssl_certificate_key: /var/lib/lego/certificates/proxy.kill0.net.key
|
||||||
|
raw: |
|
||||||
|
auth_basic "Proxy";
|
||||||
|
auth_basic_user_file /etc/nginx/proxy.htpasswd;
|
||||||
|
|
||||||
|
location / {
|
||||||
|
add_header Alt-Svc 'h3=":$server_port"; ma=86400';
|
||||||
|
}
|
||||||
|
|
||||||
|
location /loki {
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_pass http://loki_backend;
|
||||||
|
proxy_set_header Connection $connection_upgrade;
|
||||||
|
proxy_set_header Host $http_host;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
}
|
||||||
|
|
||||||
|
location /prometheus/ {
|
||||||
|
proxy_pass http://prometheus_backend/;
|
||||||
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user