alertmanager: configure receiver secrets

group_vars/monitor_servers/main.yaml

@@ -8,6 +8,8 @@ alertmanager_web_external_url: https://monitor.kill0.net/alertmanager
 prometheus_web_route_prefix: /
 alertmanager_web_route_prefix: /
 
+prometheus_file_sd_config_d_files: []
+
 prometheus_config:
   global:
     scrape_interval: 15s
@@ -16,6 +18,10 @@ prometheus_config:
       region: dallas
       provider: linode
       replica: A
+  remote_write:
+    - url: http://localhost:9009/api/v1/push
+      headers:
+        X-Scope-OrgID: kill0-net
   alerting:
     alertmanagers:
       - static_configs:
@@ -177,6 +183,77 @@ prometheus_config:
       static_configs:
         - targets:
             - "localhost:3002"
+#    - job_name: process-exporter
+#      scrape_interval: 5s
+#      static_configs:
+#        - targets:
+#            - "localhost:9256"
+    - job_name: loki
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - "localhost:3100"
+    - job_name: promtail
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - jump0.kill0.net:9080
+            - mine0.kill0.net:9080
+    - job_name: gitea
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:3001
+    - job_name: karma
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:8080
+    - job_name: kthxbye
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:8081
+    - job_name: smokeping
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:9374
+    - job_name: mimir
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:9009
+    - &snmp_job
+      job_name: snmp
+      static_configs:
+        - targets:
+          - 172.16.100.1
+          - 172.16.100.2
+      metrics_path: /snmp
+      params:
+        auth: [public_v2]
+        module:
+          - if_mib
+          - ip_mib
+      relabel_configs:
+        - source_labels: [__address__]
+          target_label: __param_target
+        - source_labels: [__param_target]
+          target_label: instance
+        - target_label: __address__
+          replacement: 127.0.0.1:9116
+    - job_name: snmp_exporter
+      static_configs:
+        - targets:
+          - localhost:9116
+    - <<: *snmp_job
+      job_name: snmp-long
+      scrape_interval: 30s
+      scrape_timeout: 30s
+      static_configs:
+        - targets: []
+
   rule_files:
     - rules.yaml
 
@@ -228,6 +305,10 @@ prometheus_rules_config:
           expr: up{job=~"thanos.+"} == 0
           labels:
             severity: critical
+        - alert: Down
+          expr: up == 0
+          labels:
+            severity: critical
         - alert: FileSystemUsage
           expr: ((node_filesystem_size_bytes{mountpoint!~"fuse.lxcfs|tmpfs"} - node_filesystem_free_bytes) / node_filesystem_size_bytes) > 0.80
           for: 1m
@@ -280,6 +361,13 @@ prometheus_rules_config:
             # summary: Certificates expiring in < 14 days
             summary: "{% raw %}Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}){% endraw %}"
             description: "{% raw %}SSL certificate expires in 14 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}{% endraw %}"
+    - name: snmp.rules
+      rules:
+        - alert: PortDown
+          expr: ifAdminStatus{ifName=~"(Gi|eth).+", ifAlias!~".+laptop|notebook.+"} == 1 and ifOperStatus == 2
+          for: 1m
+        - alert: PortFlapping
+          expr: changes(ifOperStatus{ifName=~"(Gi|eth).+"}[5m]) > 2
 
 blackbox_exporter_config:
   modules:
@@ -309,34 +397,6 @@ blackbox_exporter_config:
       http:
         method: GET
 
-#  route:
-#    receiver: pushover-receiver
-#    mute_time_intervals:
-#      - quiet_hours
-#    routes:
-#      - receiver: blackhole
-#        match:
-#          alertname: MaintenanceMode
-#      #- receiver: blackhole
-#      #  match:
-#      #    alertname: QuietHours
-#  receivers:
-#    - name: blackhole
-#    - name: pushover-receiver
-#      pushover_configs:
-#        - token: "{{ vault_pushover_token }}"
-#          user_key: "{{ vault_pushover_user_key }}"
-#  inhibit_rules:
-#    - source_match:
-#        alertname: MaintenanceMode
-#    #- source_match:
-#    #    alertname: QuietHours
-#  time_intervals:
-#    - name: quiet_hours
-#      times:
-#        - start_time: 03:00
-#          end_time: 15:00
-
 alertmanager_config:
   inhibit_rules:
     - source_match:
@@ -345,9 +405,13 @@ alertmanager_config:
     - name: blackhole
     - name: pushover-receiver
       pushover_configs:
-        - token: agwd6wv7xveakykb8e5rz7rw3eg2v3
+        - token: "{{ vault_alertmanager_pushover_token }}"
           user_key: 28G1x3lT4oUtlck50R1H3e6j8kDHjb
+    - name: discord
+      discord_configs:
+        - webhook_url: "{{ vault_alertmanager_discord_webhook_url }}"
   route:
+    repeat_interval: 24h
     receiver: pushover-receiver
     routes:
       - match:
@@ -359,6 +423,8 @@ alertmanager_config:
       - receiver: pushover-receiver
         mute_time_intervals:
           - quiet_hours
+        continue: true
+      - receiver: discord
   time_intervals:
     - name: quiet_hours
       time_intervals:
@@ -419,3 +485,25 @@ karma_config:
 thanos_bucket_config: "{{ vault_thanos_bucket_config }}"
 
 kthxbye_listen: :8081
+
+smokeping_prober_config:
+  targets:
+    - hosts:
+      - dns.google
+      - vpn-home.kill0.net
+      - ping-home.kill0.net
+      - vpn1-sch.corp.nmi.com
+      - gp-chi.ops.nmi.com
+      - gp-ash.ops.nmi.com
+      - 169.254.0.2
+      - 172.16.100.1
+      - 172.16.100.2
+      - 172.16.10.16
+      network: ip4
+    - hosts:
+      - dns.google
+      - ping-home.kill0.net
+      - fc00::ffff:169.255.0.2
+      - fc00::ffff:169.255.0.16
+      network: ip6
+

host_vars/jump0.kill0.net/all.yaml

@@ -39,21 +39,83 @@ wireguard_interfaces:
 
 restic_tidy_enabled: true
 
+nginx_htpasswd_files: "{{ vault_nginx_htpasswd_files }}"
+
 nginx_vhosts:
   cavicc:
-    - server_name: cavi.cc
+    server:
-      root: /var/www/cavicc
+      - server_name: cavi.cc
-      listen:
+        root: /var/www/cavicc
-        - 80
+        listen:
-        - "[::]:80"
+          - 80
-      raw: |
+          - "[::]:80"
-        location / {
+        raw: |
-            return 301 https://$server_name$request_uri;
+          location / {
-        }
+              return 301 https://$server_name$request_uri;
-    - server_name: cavi.cc
+          }
-      root: /var/www/cavicc
+      - server_name: cavi.cc
-      listen:
+        root: /var/www/cavicc
-        - 443 ssl http2
+        listen:
-        - "[::]:443 ssl http2"
+          - 443 ssl
-      ssl_certificate: /etc/letsencrypt/live/cavi.cc/fullchain.pem
+          - "[::]:443 ssl"
-      ssl_certificate_key: /etc/letsencrypt/live/cavi.cc/privkey.pem
+        ssl_certificate: /var/lib/lego/certificates/cavi.cc.crt
+        ssl_certificate_key: /var/lib/lego/certificates/cavi.cc.key
+        # ssl_certificate: /etc/letsencrypt/live/cavi.cc/fullchain.pem
+        # ssl_certificate_key: /etc/letsencrypt/live/cavi.cc/privkey.pem
+        raw: |
+          location / {
+            add_header Alt-Svc 'h3=":$server_port"; ma=86400'; 
+          }
+
+  proxy:
+    upstream:
+      - name: loki_backend
+        server:
+          - localhost:3100
+      #- name: prometheus_backend
+      #  server:
+      #    - localhost:9090
+    map:
+      - name: $http_upgrade
+        variable: $connection_upgrade
+        content:
+          default: upgrade
+          '': close
+    server:
+      - server_name: proxy.kill0.net
+        root: /var/empty
+        listen:
+          - 80
+          - "[::]:80"
+        raw: |
+          location / {
+              return 301 https://$server_name$request_uri;
+          }
+      - server_name: proxy.kill0.net
+        root: /var/empty
+        listen:
+          - 443 ssl
+          - "[::]:443 ssl"
+        # ssl_certificate: /etc/letsencrypt/live/proxy.kill0.net/fullchain.pem
+        # ssl_certificate_key: /etc/letsencrypt/live/proxy.kill0.net/privkey.pem
+        ssl_certificate: /var/lib/lego/certificates/proxy.kill0.net.crt
+        ssl_certificate_key: /var/lib/lego/certificates/proxy.kill0.net.key
+        raw: |
+          auth_basic "Proxy";
+          auth_basic_user_file /etc/nginx/proxy.htpasswd;
+
+          location / {
+            add_header Alt-Svc 'h3=":$server_port"; ma=86400'; 
+          }
+
+          location /loki {
+            proxy_http_version 1.1;
+            proxy_pass http://loki_backend;
+            proxy_set_header Connection $connection_upgrade;
+            proxy_set_header Host $http_host;
+            proxy_set_header Upgrade $http_upgrade;
+          }
+
+          location /prometheus/ {
+            proxy_pass http://prometheus_backend/;
+          }