Add role for ufw

group_vars/all/main.yaml

@@ -102,17 +102,17 @@ rsyslog_archival_format_enabled: true
 rsyslog_outputs:
   - name: omfwd
     params:
-      #target: 127.254.254.1
+      target: 169.254.0.1
-      target: 10.255.0.1
-      #port: 1514
       port: 514
       protocol: tcp
       action.resumeretrycount: -1
       queue.type: linkedlist
-      queue.size: 10000
+      queue.size: 1000000
       queue.filename: fwd
       queue.saveonshutdown: "on"
       keepalive: "on"
+      template: RSYSLOG_SyslogProtocol23Format
+      tcp_framing: octet-counted
 
 sudo_aliases:
   host:
@@ -210,17 +210,17 @@ teleport_config:
 firewall_ipset_node_exporter:
   - "{{ lookup('dig', 'jump0.kill0.net./A') }}"
   - "{{ lookup('dig', 'jump0.kill0.net./AAAA') }}"
-  - 10.255.0.1
+  - 169.254.0.1
 
 firewall_ipset_blackbox_exporter:
   - "{{ lookup('dig', 'jump0.kill0.net./A') }}"
   - "{{ lookup('dig', 'jump0.kill0.net./AAAA') }}"
-  - 10.255.0.1
+  - 169.254.0.1
 
 firewall_ipset_mtail:
   - "{{ lookup('dig', 'jump0.kill0.net./A') }}"
   - "{{ lookup('dig', 'jump0.kill0.net./AAAA') }}"
-  - 10.255.0.1
+  - 169.254.0.1
 
 node_exporter_du_directories:
   - /var/log/syslog
@@ -230,7 +230,7 @@ wireguard_iptables:
   wg0:
     input: true
 
-wireguard_network_prefix: 10.255.0
+wireguard_network_prefix: 169.254.0
 wireguard_peers:
   wg0:
   - public_key: 1ipGUnK8XDbIoBIEF440BhwLUe0yHa5l3kZZc4eFxV8=
@@ -241,57 +241,125 @@ supervisor_unix_http_server_socket_chown: root:node_exporter
 supervisor_unix_http_server_socket_chmod: "0770"
 
 firewall_ipset_loki:
-  - 10.255.0.1
+  - 169.254.0.0/24
+
+firewall_ipset_promtail:
+  - "{{ lookup('dig', 'jump0.kill0.net./A') }}"
+  - "{{ lookup('dig', 'jump0.kill0.net./AAAA') }}"
+  - 169.264.0.0/24
 
 promtail_clients:
-  - url: http://10.255.0.1:3100/loki/api/v1/push
+  - url: http://169.254.0.1:3100/loki/api/v1/push
+    external_labels:
+      region: dallas
+      provider: linode
 
 promtail_scrape_configs:
-- job_name: system
+- job_name: journal
+  journal:
+    json: false
+    max_age: 12h
+    path: /var/log/journal
+    labels:
+      job: systemd-journal
+  relabel_configs:
+    - source_labels:
+      - __journal__systemd_unit
+      target_label: systemd_unit
+    - source_labels:
+      - __journal_unit
+      target_label: unit
+    - source_labels:
+      - __journal_priority_keyword
+      target_label: priority
+    - source_labels:
+      - __journal_syslog_identifier
+      target_label: syslog_identifier
+  pipeline_stages:
+    - match:
+        selector: '{systemd_unit=~"(alertmanager|blackbox_exporter|grafana|karma|kthxbye|loki|mimir|node_exporter|prometheus|promtail|pushgateway|thanos).+"}'
+        stages:
+          - logfmt:
+              mapping:
+                level:
+                ts:
+          - timestamp:
+              source: ts
+              format: RFC3339Nano
+          - timestamp:
+              source: t
+              format: RFC3339Nano
+          - labels:
+              priority: level
+- job_name: nginx-access
   static_configs:
-  - targets:
+    - targets:
-      - localhost
+        - localhost
+      labels:
+        job: nginx-access
+        __path__: /var/log/nginx/*.access.log
+  pipeline_stages:
+    - match:
+        selector: '{job="nginx-access"}'
+        stages:
+          - regex:
+              expression: ^(?P<hostname>[0-9A-Za-z\.:-]+) (?P<remote_addr>[0-9A-Za-z\.:-]+) (?P<remote_logname>[0-9A-Za-z-]+) (?P<remote_username>[0-9A-Za-z-]+) \[(?P<timestamp>\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} (\+|-)\d{4})\] "(?P<request_method>[A-Z]+) (?P<URI>\S+) (?P<http_version>HTTP\/[0-9\.]+)" (?P<request_status>\d{3})
+          - timestamp:
+              source: timestamp
+              format: "02/Jan/2006:15:04:05 -0700"
+          - labels:
+              hostname:
+              method: request_method
+              status: request_status
+              version: http_version
+- job_name: nginx-error
+  static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: nginx-error
+        __path__: /var/log/nginx/*.error.log
+  pipeline_stages:
+    - match:
+        selector: '{job="nginx-error"}'
+        stages:
+          - regex:
+              expression: '^(?P<timestamp>\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}) \[(?P<priority>\w+)\] (?P<pid>\d+)\#(?P<tid>\d+): (?:\*(?P<cid>\d+))?'
+          - labels:
+              priority:
+          - timestamp:
+              source: timestamp
+              format: "2023/08/16 02:43:32"
+          - regex:
+              expression: 'host: "(?P<hostname>[0-9A-Za-z\.:-]+)"'
+          - labels:
+              hostname:
+- job_name: syslog
+  syslog:
+    listen_address: 0.0.0.0:1514
+    listen_protocol: tcp
+    idle_timeout: 60s
+    label_structured_data: true
     labels:
       job: syslog
-      __path__: "/var/log/syslog/{{ ansible_hostname }}/**/*.log"
-- job_name: nginx
-  static_configs:
-  - targets:
-      - localhost
-    labels:
-      job: nginx
-      host: "{{ ansible_hostname }}"
-      __path__: /var/log/nginx/*.log
   pipeline_stages:
-  - match:
+    - match:
-      selector: '{job="nginx"}'
+        selector: '{host=~"ap0|coresw0|fw0|power0|172\\."}'
-      stages:
+        stages:
-      - regex:
+          - static_labels:
-          expression: '^(?P<remote_addr>[^ ]+) - (?P<remote_user>[^ ]*) \[(?P<time_local>.*)\] "(?P<method>[^ ]*) (?P<request>[^ ]*) (?P<protocol>[^ ]*)" (?P<status>[\d]+) (?P<body_bytes_sent>[\d]+) "(?P<http_referer>[^"]*)" "(?P<http_user_agent>[^"]*)"?'
+              region: home
-      - metrics:
+              provider: home
-          nginx_requests_total:
-            type: Counter
-            description: requests in nginx access logs
-            source: method
-            config:
-              action: inc
-      - labels:
-          #remote_addr:
-          #remote_user:
-          #time_local:
-          method:
-          #request:
-          #protocol:
-          status:
-          body_bytes_sent:
-          #http_referer:
-          #http_user_agent:
 
-loki_service_enabled: false
+  relabel_configs:
-loki_service_state: stopped
+   - source_labels:
-
+       - __syslog_message_hostname
-promtail_service_enabled: false
+     target_label: host
-promtail_service_state: stopped
+   - source_labels:
+       - __syslog_message_severity
+     target_label: priority
+   - source_labels:
+       - __syslog_message_app_name
+     target_label: syslog_identifier
 
 influxdb_service_enabled: false
 influxdb_service_state: stopped
@@ -300,3 +368,7 @@ influxdb_package_state: absent
 telegraf_service_enabled: false
 telegraf_service_state: stopped
 telegraf_package_state: absent
+
+lego_credential_files:
+  - name: credentials.json
+    content: "{{ vault_lego_gcp_service_account | string }}"

group_vars/jump_servers/main.yaml

@@ -7,7 +7,7 @@ firewall_allowed_udp_ports:
   - 1194
 
 firewall_ipset_syslog:
-  - 10.255.0.0/24
+  - 169.254.0.0/24
 
 autossh_authorized_keys:
   - key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOvKqDI6VUYFgMUC54pVr5U8CX+Xl2ewV7PIYkTiQ70o
@@ -46,7 +46,7 @@ telegraf_config_d:
           name_override: ping6
           binary: ping6
         - urls:
-            - 10.255.0.1
+            - 169.254.0.1
           count: 10
           ipv6: false
           binary: ping4
@@ -311,43 +311,49 @@ wireguard_peers:
   wg0:
     - comment: mine0.kill0.net
       public_key: Cm9yZNczjghAh4hV4fSvy3rsmuLsQFZk+ET5CoWxVnI=
-      #endpoint: "{{ lookup('dig', 'mine0.kill0.net./A') }}:{{ wireguard_port }}"
       endpoint: "mine0.kill0.net:{{ wireguard_port }}"
-      allowed_ips: "{{ hostvars['mine0.kill0.net'].wireguard_interfaces.wg0.address }}"
-    - comment: vpn-home.kill0.net
-      public_key: j5AgKWcXx8we7QVkt6//oQWsGfXj+5IJKt9mx0EpTW0=
-      endpoint: "vpn-home.kill0.net:{{ wireguard_port }}"
-      allowed_ips: 172.16.0.0/16, 10.255.0.2/32
-  wg1:
-    - comment: pixel-2
-      public_key: GzQOU0x1POvkY4+6smBGkE/B1XytoVxIJa6zGX8j6Bc=
       allowed_ips:
-        - 192.168.255.16/32
+        - 169.254.0.2/32
-        - 2600:3c00:e000:343::10/128
+        - fc00::ffff:169.254.0.2/128
+    - comment: vpn-home.kill0.net
+      allowed_ips:
+        - 172.16.0.0/16
+        - 169.254.0.16/32
+        - fc00::ffff:169.254.0.16/128
+      endpoint: "vpn-home.kill0.net:{{ wireguard_port }}"
+      persistent_keepalive: 25
+      preshared_key: "{{ vault_wireguard_preshared_key.home }}"
+      public_key: fUSQ7Uxkxij/0p+SIRekb6moqW0t/qdFaP2HsjRsNRs=
+    - comment: retropie
+      allowed_ips:
+        - 172.31.0.0/16
+        - 169.254.0.17/32
+        - fc00::ffff:169.254.0.17/128
+      persistent_keepalive: 25
+      preshared_key: "{{ vault_wireguard_preshared_key.retropie }}"
+      public_key: lLvracXkf8HNfgKpJkzei9ys58aAs4DT3Z3bjNRFsQY=
+  wg1:
+    - comment: pixel
+      public_key: zCDfH5Eqv0oRNWC8TtrkGby3+BAtiQtXxbsmA/lZtXQ=
+      allowed_ips:
+        - 192.168.255.16/24
+        - fc01::ffff:192.168.255.16/128
+        - 2600:3c00:e000:343::ffff:192.168.255.16/128
     - comment: work laptop
       public_key: TRT1SRQd3mFJDJK9tdglqsydXJmkzyrNdUOm4nr7M3k=
       allowed_ips:
-        - 192.168.255.17/32
+        - 192.168.255.17/24
-        - 2600:3c00:e000:343::11/128
+        - fc01::ffff:192.168.255.17/128
-    - comment: home workstation
+        - 2600:3c00:e000:343::ffff:192.168.255.17/128
-      public_key: ISvgu8zZWjmKyKrJi2mbqoJg2mrvIjPbQRs0Sp+dLzc=
-      allowed_ips:
-        - 192.168.255.18/32
-        - 2600:3c00:e000:343::12/128
-    - comment: rick
-      public_key: oFJcRhs7tQ4vPHTjbKwwWirpjx9T9ng7PFj3+iAVYWo=
-      allowed_ips:
-        - 192.168.255.32/32
-        - 2600:3c00:e000:343::20/128
 
 unbound_interfaces:
   - 127.0.0.1
   - 192.168.255.1
   - ::1
-  - 2600:3c00:e000:343::1
+  - 2600:3c00:e000:343::ffff:192.168.255.1
 
 unbound_access_control:
   - 127.0.0.1 allow
   - 192.168.255.0/24 allow
   - ::1 allow
-  - 2600:3c00:e000:343::/64 allow
+  - 2600:3c00:e000:343::ffff:192.168.255.0/120 allow

group_vars/jump_servers/vault.yaml

@@ -1,223 +1,230 @@
 $ANSIBLE_VAULT;1.1;AES256
-36396137393836323465386631643461656431316666376562623633383965393863383866663764
+34326635363163333038303363346632613636306133616266343732323036656335643366646264
-3664343734343065343236303365373962333162306564620a623362326163393766343735653061
+3938363837343132633665323362323133663430633165310a303562396164626233653535623336
-64393932383066323264636530613036353637343231666439346234663430326366396532663765
+34646463376565646435616564616235663836663466353234343030353363626131613134643431
-3536663666643838360a316462376363613562373965653536333763386635343362393938386331
+6535653237343635300a393162633862323261376530396630643539313162653161396438366236
-39663266616365383166393232646530656135373234646166393365343233666635393430313136
+39633866303562393131636537653932306138643766653632323834373361323938393131656331
-66616361636638323430343334643230623331623334343162333335353265333436326239626664
+64653335393632336533343135313766643361633739613333666461663962343134636263333333
-30623039333737383531663738616337396136353836383537343337316565623562393235303566
+30663966306434323331373136366333623262393962363031353564383133306433306261616631
-63656234663765313062666435313431633861646137313330386633383062656335336639633631
+39323738373163653861653866366139346666333338303435333435663532343466393561616230
-31386561376365623634666231643134663230643736376662356361313464666638363961366437
+31656234376564366533663762366639363134613666363532336463613863363862353839313034
-61323033386661356561653961623333353637613439666437333164643532343863333434613061
+32343938656461643531373535363837663336303137323766663966613136313365333734366233
-63646432396333303965663730623061333065653432326136333337633862393339363130373138
+32613630343034356136313661616532356163336561633562386337613937616535306533623838
-36366163316635383336316537393761633962336138643139386638373134313635336666303765
+31666363336363653436623635303231366364343137343532613263313436356365393330666638
-62316531336165323965343232636339313462633536623139303865663862376364363261363865
+65383161613561343361326431623338356338323164656536306162333764346131623235633664
-31353064646338646662386639343462386639393162363334363937363337613963313135663365
+64666635343765316134653936666137613465363735316562616336636233383439653564316135
-66343365363232623564613035303139663937356430336537346564643134313763393462323638
+61623466373965323437306537313761353832376462396465306532356162643966643534633666
-30616462363661623466663162333834323937623335316261646533316137613564316532653165
+35643066653166313335633737393362353630623639336366323161666232353930396434333630
-33343133376538643961656364656666346533316336626464663939313137643461303232666162
+31353232663837393764653465303133616265636132316430393936323735663136383539336462
-32353131353864373738396335613763366639633837653636386139393862616364613265313935
+37333262373738366266653532393937326163363832356438373635646465646230623738633232
-62353134303733393836666337393530643465343333373230346133396163623332336131323730
+61626530323834383838333861363335613034366661343138336638323432306135356363353330
-39383264303935343763343033303864316433613334633137333031626563393233663932376434
+63396538663731383637333763663763376361313739366266373065303230373135653831643735
-66303638643232376633636331613234316339666630393534333136306639616662613361663031
+62356365653935386130643364393963353335633539663061633838373132633336613664356631
-31316630323338383061346333633063393261353463623039633063633132623730303161663531
+65616639643461666538653334666465393965663862343530656265663032653561343833336563
-65353030303763336639636265663333333639306432306662386232303439626235663433376437
+31653533383665306166393431626161363364346265643631373366316434336234653264666164
-37336461376662663035373336663937333132383964396561626337626632303064656365313633
+32373336326434666561383463383037633338646635636364366563666464346433643064323032
-61663630316163323163383436636636313333353437646330346532656236626562663332323636
+66313065303638636635353864613238346537386131303666386264376561393134613438316239
-65303430663133363464323262313531376531303739613364336262393965376533343136323034
+30623238356663393632326531643732313433383638333866363161656534393134313937383161
-65376461326362313732323730353137663036393835333939353962643338326162306163626536
+65306439393965353461363439336165356562323262633664653231633538386661616238303732
-37316262623265633363356435316632653466636137303131303664636433376236613237376339
+37623964613335393330663862666135666664353134303861653232623730626533616335643539
-36616639643232356330393134333364303137633736633764346233636330386232316566366435
+62396361356465323165366235303362383736386664663935353666613132663762303238346533
-30613261613936343738303763623966653936323661383164613933333633653339363535306138
+38303665333639323336643466353637636364643631613231613164303664336462353831363662
-32326466306634633965666466393435656432336163663130666266363230653730396665623531
+33373865326563653632643131313330663237636135376563336565633162613033356163663333
-36643364306537306663303537333063363565386337663061623661343838303638393965373165
+37383231306333343436366535396463636130353663303830343933623135343661653030643438
-38613939613061376161626163336164656237356164303562376137633135613738386331323262
+36363663656138326435313565383864373036653832663163633236363961303238346234633231
-30373539633630646339323930373737346136633465616535643439643134306430653062383664
+33653235643666353266316463373665633661333262303764346466636639316138656266656235
-61313138376138373961376561303162616438663263653561363339396132393834373566663436
+65353936356230613130373339336631396639303533366239363037626365653262353563643334
-62356331323465616134656237356434633830666231646434363664623139373737393830616338
+63623537663966353332383838653939653062663864396235633232376635383035313961386638
-36353066613464353739336462623966356330653534366332663735663937306462393233383939
+33623062336630653432663234303561663233633566343862303631663337383834393930666537
-36363066633563393463303363653631646464323937613234333835306139373462366661643961
+66376633303034316435366237366464366336313932666337356664323265343533306230343332
-30316462636638353531336266633061663933316266303335623837376239633835663265336338
+32366239643033333635343563353437633439663839613733636339353933613762303733343736
-39313334396565653262613736616536646461656438373839316337363963663135353261353133
+65633937653161623732393137313062393636373461306265373461396538663937623263323630
-32373366366236353663393065306338373961636432353533386436666532313637306433373236
+65626230666636336233303166666664366361366534386466393337373162646262356138636433
-38383037663037643763383465313862336334326637346338383235663061316232613365656266
+32346238643937343865653165326566346531626238643434623765353836653061623064653166
-31616136373135323039313633373538353761663439323839313365313462663063373339623530
+62396531333937393363633835663930323138656365313865373733636135333735656138353030
-61313731303861333631613464343232303763316462643935626366346130366531313631626630
+64313461356232633065613139376134303433613663653733663266376437306337396662353130
-39636630663866336161623835666261366337376239653139613230616231353636616266663238
+39613732666566636434656466343839353634663736636636666231336235396439393961313366
-31653466363530346262326630353661366635616162313733323032633736653362306665363565
+65363130666635663633646663656430386538343931346233396563613339333331663930306132
-31653731343465373736646338383830393735643736646266323965356336393939366537386566
+38363034333434633933303862383965303835343961343562346466393466393165663965343936
-35613561333834653834626233396133323337303439643432373931616237613439343665343061
+62316234663738356361393836363939393962616639306366653934386539373736636233623763
-39666661353532326435373332393739356636636433623163383337663165613834393864303533
+30643165353665313235373366366164343461616238313239313737626465653930366466623164
-32356336366336353261653235663666633335626331663964636263656136366232373838613962
+38653533346335633437653237613436333463373163646261376264376438656131366263353862
-37393464376137663630333334363234393464313062353366656435646633653265616265383535
+38386361346438343036373761383164666465663436363132373662343266666433383663663333
-61333061303633623065666366643037333139356465343932376664333163623532626331336139
+31326434666136623865626635663232333766343538383839303435646439386133613663373736
-33373732613264636331623964393336383665613264343131613138386362386362343539346234
+31373664353630313461363162663866333366613666646337363761333237393635393864373531
-30336237356436623262393139363538306530356530353237666339386565613931303131666262
+33386434386536343033633664373963323937646535373231623836396334373431353964386566
-30363866393061663437633532356238383530363066623862393531366530613731393137343434
+31633065346534323566653734663261353866613635316165336534666134653439613463323031
-33386434613632383066636638356161323837653630363830336233653830343261303933616565
+63656435643132633664393234396230396336326139386632303633393130316566353834376135
-65313334633838663264623032656131646331613539666436343334663061313837353030626161
+31373663326665333164626433303938666366666463643134356236613738636434626665663461
-63303362666662356235343065373231646334656565316564626234363431346664373036303333
+66376665363633393530616365643139313436383137323062383763613931353330643634616236
-39343363346365323237356365323062313630323736323737643432353262366534653131313033
+31323131666536613433396538643364336562366433623437336564663638333136313531623761
-63383638333334333361383461626361333766343861653538343562326366623332626131613136
+35636431383562393237663533333161333933643662666635623965386435356534633832373531
-62643537636233383263656564306430386333346432353434623433373638366536393438333434
+35343132663861313931636530666237353166633031366330643731663561346133373831633137
-37656539303736633938316462366230613131633936363034386639623330653535326264333861
+30633332633362396664333736613630346437353836613237323835313730333033343430323236
-35616537623461316662636166613530373963316236393938363932616566333430613366626363
+64373663653563343838323438396661363839623261663339333062656264323866386536633439
-66383139323565353830303466356233353066316663653732303534383765346636653132363130
+39346532633864633663356431663535343664376265376566653861616434313665616264626230
-32303563353232616537613966663836623832383335646331616364353336313363313234323362
+33316134386630313139343030393435626564353666343734376561616437343032306566303031
-66616136636533346339363563623734623239626230636565623338363861393338613337623530
+32353663653537666137343831633164303934303436356161313661613164666431653037363539
-64626363343533303333626234326666623136333332323532383662663635633538313433303835
+65326366323033366663623736626366613239323033356566383334373434313636336230643639
-34623134386631376639623639313164393033616664346338633033656630623436633130373665
+63646131343636303262626230653633393735323030373531346437396663313162623332316362
-38356635396238613633333738326233663933666562356630613063303230353462653264393531
+34366239326366633961363236313930303435646135366565626564383663306636623034653465
-31303736633030663761376134366631646130363139623465653661366335363830633566333237
+62373539663561366435356538386664373664653239313936623362326636353563343337336632
-33376631343334376435386135653330343832353339313931323434303265343361336231643638
+31333133383562653935656265363136363532653431623830396130636233306563623663333531
-66623539313162643337353432393865626538633265633363353830306663393233333962313636
+38383664366363306662383532656366356266323031613630336338656362643562373034633933
-33333565356536376464653131376633353363316663336563323230326537613165353134366365
+61623865316636643430653562623535643966306265613833396266626564326161383666616263
-61363030326334656139353938613531643864316434383266353633373735326562306239323961
+66663664303431353866613237316539343835366531363166633136633965386532613831346566
-37336638663837333738313230316236346262326135346536343331356234313130353661383464
+35313334356132626337633339363166303637313665303464343635323163383231636238613066
-35376236346366373363326138383430323132626663303138353938383263643665393839363162
+34613462386533326638643764346661346361343166376337353136313361656561396238626538
-31366166613037383166313264373035663066336138623535313035303533613132613436313136
+61666431636661643665323330643239613734663332336638613435653563303835306639316162
-66393764333732356333363462333366346363613262316130636235353361313731383839653563
+39363432643364393036333334643430663763363234666463323231336135343763653063343533
-63383134643262636262666237356233393430336163613135623264633336396139646231363562
+32373862383062346261646331376633316463393365303931303535373137663561396636323633
-34393031663961643562396234666437356665356331633834396637336264653265353065306233
+65626533383337393838323963326361623663386639656264366662326262653161336661306137
-30393461313663313564373236663362353435393535306465353136613730333866636639633161
+64356561623164303465633562393462396166316233633561323565666433376565646534346132
-30666566393266616134636264366666356438616632336661393639366635356262653832353633
+34343862393766346534393662316336393363363937313765663237383961356266656233623432
-32623466303835633065613936373063626432326463336163303838613836646332643035653933
+65383465633830393064393262343133376161646239663166393339643034343635343265636233
-63363630663161373039653330633631643638313036633537323364373739363736656231636535
+64623664653538343961326663626365333533613338366332396437616466326362346463656465
-35396466373666353361366535366334313538313639663131336662386166316162326331373838
+30323233343564396238613038663835353538336163333933373538393766633532653736613165
-34386232653930383133613164393435346661643466343762343463376537633036393366656164
+39343938373535343135656430663263626366346535333833393566363938306430396664623864
-34366465613839623533363235343737333565326165633634386230323938646166643737333261
+39303539373262383438356566663736623364363766396238323730306263373639303262376463
-64333139663463666432346461613033616539643463323263343563303361373539303834353434
+63353066306534313031343933343632613634366565386230636137653530393334373832646339
-61306635323463383238633738303830646263663036396566336534623237636234303566643533
+39396535336466336364666461383639303433383563343236366336316637353032316430646362
-39663462663063386137326630353164633561653936343665326665306665326238303230346436
+65326339383635333666396233323539316664343031613333653133343732303335633131633031
-31633138303236666362306162663036386334623339656565353730643630396263363738306139
+66353338363535323734623332633939343230363761646461356534343030326161353131313963
-64323230616164303638643263396432646438356534313433633536656432333738303038323266
+30323331393133366330653862396265343938623366366164633534653538613461326139353436
-31643965383036326134653030333932323231313363336263656534303839346631636230323032
+32353939633536616663333763393532323765353533633065373064613438383566373264353362
-61303033383932626238353466353631326633633565343065306561396636393835373966383032
+37396137353464376362656662303530343261666530663931383031363830356234393162336131
-61363061653662373731313862326461373133343930393963343062623663633033323865323565
+66313339623064623233393130616532613038623636393035623935346565393061633566663062
-62633736623365613631326464373662393861663737623836666532353339363232363630333662
+65663563356230316665363863373839326464303632333136643136323334663263343561663530
-65333265386561336337353838353238316466336162393738623034376339653864393733643837
+33363763393463373637366462653036336461366264333433393366316438343565656232616133
-38313763656431323261366634386331366262653838613036646633326464383565353136356566
+34333762656562353734383833376234383161396263613534313736346330666237343937313661
-32313131313466613266643435663933646132646339353239343535363333393535346565383331
+65613631323966393666323834323564356437313032633830616163656365353539623031313762
-32326566383337323662663438316639366139386433316639633463333661396337393837646435
+65323266626366666366396161373562633938303361396665663536316236333236383234386432
-66313637653939626536326332306139393438333137323532316130636439313066383633396335
+37666336663362623365343632353734623131346636653539316635336265303137323064313032
-38373062353930623661306339653234336135396233383965303861363535616633366666656562
+33613036343231666232306233623266663466656362316439643263643163616139303939393430
-37336331316534656465613536313364346633393066323839393833393864363234356330663264
+63663332626161336637626433386264613131363933313937373030396262343238343565363161
-65336263613861383837373533646430666539316638323966623761373633666437306432386235
+33666365343534656366366430646639656664656534643831346136643064383931396430383966
-66353531303533323662613565363065356236383939623237363835616262326536373962343538
+36653166353766656262333434303436643339346365613239386630363430613465366632383733
-30316631656465313264393932626232346637356531336536613561383434663934643432613164
+31323737616236633535613030313564656364363234386634383234393639313366323333623764
-33313833613532613365393637323262346437343933353138623765626665656663306263393862
+31353861653964663764633332656133316562373164633433623266623531343663643939633236
-39303865316537643063363665626465356631653534393462353830653931636563653333323733
+64333635303637653337353164326237316262656237636236643335633331303532353531346531
-31343864333630366566613731366333323631313337636236653662613832626464626333363537
+64643765353735333634303936356131613866326335376331393733326633653536333563326530
-33303762363332306266323538323366383863383033616563376231303937316163396638663162
+37353566343236393832653964656262636531376464646433656364353738363762323661646437
-64386664313863636535366331646238626437353664313731346633353738343733626263666230
+33623234343565646539316361663331623133323238393264613566633930346561613533353862
-30616161333061393061366430656330613737333133656637656664316265616365313436373939
+38353336623131366331336535626132636638393337376236396462333839363764653264653837
-65653564326165303761326236343436326363383538613734303539363363316135653630666138
+34326265376538353833343830653431646464643762613661303963363534656465363564366139
-38663333323863363163353838653765353937313166316230323961376136326438653866346665
+35646461616263646365303232396331343532626635303631313934656332393837616264306234
-34306561356536663363666162643362316139313438323632366136366461663230613563613434
+37313966656462353161363661386336636363663437346532326361613864353961366432356237
-37333838663239356236343731313430363232623633626364336664613839393036393566656366
+37386536393866326662343334353237633436383235633636383666613136386465316363393939
-61616332666262336231363262333832613937313330373231383830343130323966333261353661
+32303138643761653735323037346464653635366430356336313966643537646135623938613033
-34633661363731613430393262373839333863393730613730323866623837363936333039383535
+65373835303539383830643838383231363735383938373638663165623966356662396665303032
-36353763313565633037393032386135376537343430363535376238376131653935366434346431
+33646564306334336663636165303633346131373239316564343631306437383462303961626432
-33353338323935613638306234353963653438323031643735613035613335393834343961373037
+63396263653039336134343530653639356466616331306431633635376364613765663464346433
-37653131333336353230636136633431333463316137333636363338333230656131346633326162
+34333332663766383838653535643765383761363261326233643832353334386439396263336363
-33303635613033333730663162623965343230303533393065306539666439656361306634646662
+37336362313062616639663731363038633634383937373034656664626436383735613139393163
-38616234326637393364303731303566363661633462393836633237353139616634373933356462
+62353933336431356633346166356166616632373035363366393231383232353831633061333833
-66303864333133643238313061386538313430636231653265336463633437396134626238386365
+39316538636662333936373731363531663562623931643761353566343662363236356231323934
-38646135363764373837376534386132616139396238373765316633336135396462646230396233
+38343232393932313837323636383763633664643561383936653235303635313532333862633836
-38393432373736343236646364313037633032666631313462356164656465333837383037353038
+36303865366132316337623165396264613565323937316166653566653738343838663932646463
-39343962646236363633323465636638656266323966393635373163323330613937656266326636
+31623361303230343037386133343065633633316265633739643137343939663339656165306534
-64633666323061623266643939366630396237643731343531623031663363663963376336316334
+30346437666261323336613264353231333936633031653235633831396263653139643637663761
-31323836366665386336313139613836353764343066633231306433363538393438366162376537
+32643436396534643766316364666339613732313132356663613736623333653861376331626663
-38306436346662336262623832323964663138383262393262396366656465343731373135663562
+65636136303938376531323431323231363662303462353232613963373764616137333832383033
-63316230366236376238346639613034656662623166306536303031313930343938363363626333
+65633262313662383136646161323231643836313363383333616637353838333361663237373232
-35353837326134646535626164663762306431306464323230663763616465636435643064393830
+36626661313039613632653261636333303731396232346536666563326465393637383366383130
-65663439343166376163346137666431653731313738623630623263643133353439363730623230
+30306139383233343965623064353238316138336139363161616234643865366366336135346430
-34303265383164623530366334343066316361313533323831343833623634326661366532313265
+62393638376539643564343065396539313264396236613032306464346461613832663536373336
-64333034636663383437666238346434313761366262626231666163373433343230623662653762
+61633336616264353265313336353262646234316338626362653236346565646339663733363230
-37363234623932636536356565313062633131313334623364333262336561616334643534316666
+37393562383137336636383765363066636363373632613265653837356564313435303932333062
-38623032376432616339343939646638303630326235316163363530326238306335656630336462
+32393436343733383963336337613662666561336363303632333035346633386339303965333861
-36313234643064333737613661393164306263353438666334646164346430333665396665386436
+39333839613030326163336566623239323261346239353438303337316162353066343031303363
-32643136323431303063306135363131373966343666616163326466656233386532383930343764
+37383564316664336432303834653736346539306562663165313464356631663537383761323836
-34313536643663623031326236663866396165656539313461313933343035306336643631363261
+39363530393461666535306332333632643162663136323337323234353036623835343638333035
-65333934333231373435376134643237343237636230386465663832363665333334316663303761
+39373464633538393339626363633132343831653730376535623232653662613065326463313464
-32616133386637303437376639316261643938383563636433633035353138343137623838313466
+39323037643537626638343238343030386336326235376439313934313438653665643238366463
-65643835643562303234373137323037643165393738366262633638323939653233666163646630
+63393435643638353662333465396331323838313032653736343639373838336664633761323839
-31613863393832336663326266306430663864323031383161663762636535636238363663343066
+33663563366461313964363465373531386561613331373935363430363935363436643139616365
-38306533663931623537363964323733666563663765656331306236353436646566343766313039
+66346635333233313464313034643432383763616235326538363464303366636565393736353230
-37646334643839326531326132633433653030376437373734643038653732346335653161323932
+66356162373862383338346166333030616565643930626261623733626665333135626564623237
-36616533346437373665636166313337353136616466383237396266373131353136313535323666
+62393766313663366537306261613536356264303063383037626636366465653431383838313963
-63373034613961643531643936633566383231336166323762316539373334323134636332383232
+38666536613438333935633966643866623737646335323239613666316634613065323134303630
-36383336656538386631393665336661393432373339323432636565613963656232623034656635
+32313661303735613336373937396532353362306666383664376533643464303332643466383330
-63376161306631326632636232653831643636396365303762323661366166353539343939313561
+32343765633235356134626132383132306463366564323631323530363337343863316238393930
-39616233643564656538303764366365326338303436303261656433313766373766383638333634
+39356334303361306535653565653230336433646564353234633736663636333832353838363161
-66346464623565366530663163666339333636363463336564393034373564633565623535646136
+36623139666432666161313562373232656663646637326562396161633839366133623266356261
-37613133346565363230653666356631343037636638343832663866613461333061313464373736
+35373536623062306664653633343437653361333031303964353436636330353033653964313738
-37323563663634373931396232626436626533323566323463346535353362333262633764366664
+38663534376233383739643665303635613132643139346161633031623333653163343762336639
-30373337666366313866656362613562656239653565613035323936383861663931616266313637
+37363465373366386132393530326163363064383931313231646236313862383562666633366631
-31636631326630393834346237613965396534323366313039643566343133363537393632663264
+38646537643434653137613765653838383234366538653563363237663262323936646137366664
-66366265623962353164336463373031323262323936383163613834643433616333306661613430
+36383032623839316165626663623639363466666366373666326133616266663265383365663666
-62366464353464326636656234336433656633376636366139343338373161303965333637626661
+39316334663862656437303837613638643839343139663765613065323433346138396564376462
-30336337343936356131303237393264363232653033363163363036376163336639353961343563
+30366138316631343434396532313431313762636330653936366161623561643035356434363936
-35346336666335636266373861626465633733613032393438616434313735316132313665663635
+61643762613638316634613365623731333831616664356335613764373865623964623138643939
-34326438316632346666636265633035383336336462656331353737623066313765373366396636
+36623765333933336630666533343462313062623463646335643865356365343535643465373435
-37383366303764386566316261316232663163616234663966396665313138303839646262306338
+36623461336364373631663733613233303865353230363933333338643861313362613935366663
-63363365333735626165373735333631363761663735356635386139393739313764623531326561
+61643037326163613435373264653332386337396239393238313864316235363162396466306539
-61663936363437376261613266633163326366333730323063633436643037663631303537656363
+64643864316230363632313833326136386237366364316436346437643731393930653137373231
-66633334623064643239336439613735333431363631333435373532316230623065316332336438
+65363637316636303438343465366262353832633538343837386637376235663230336530643836
-37346336366466366335653562646265613033656466306632646566626666323337353336366366
+39633362313963643134323734313033336433663066316531303331376463653537336463356364
-62346163383439363933633763376639386132313333616261346234343439653533333462663436
+32316366393464313036666433303031633437653736303935333733373535623732373463643031
-65353165313865313635383538633432613565343136383665303064636434313135383236636436
+31383031626566623239346337616134666436616465396439343736346662336537326265353264
-30626538303437623837343663396464666232393139656335613739356165616136316263323337
+39373666383265323233376234333233346331363364633735323266376133306634373735323265
-38386537326132386264363066333730653863353430643633656533663262613963633231383533
+35636461306361353531663237616239643565633036653230333435646163376433616635393133
-65623032356131313936623931333234303532626533316636633763393631313139326562616530
+64663266383235666461666531616464373233356132333231313637396663366536666264613364
-37343965373835393564613630373632666437393738666633636536366135316336333565336538
+30333639636365626338363837623934616331353735343336656235373335616638363462383032
-61636635633861353561353063666433343837313733653837653239393061313732373930323339
+33396338346231363036613732333466633539393037326664653237643733366665356232336338
-33653965346230616336323766363434643030633166313562366561363963396663626239343834
+64626265633035386164636534613461636236306563316465333537333364333263323061393330
-34663933373832666635643961613461643331346564323431343365343439626135613638343866
+36323130376261373339613931363634386163326263303237393931616435666566393466336465
-65333732653366343032373833623566613865323539666463623163623937343338386632646330
+34396163613731613238613264316430313163666536623337376434393765356438373565626339
-34393865333864343666376265353062383966653839316263376434636531366561316433373835
+35333164333037626262626635316561323435653432613435383439653364633831616233303530
-63343264383465336439356565313130373736376532376538336533323134666565346261353435
+66656130313531316661306565313536653133303664303362643361653364383731363039343532
-62343534313866343331346439303164633539336537613130353364353430323361383938323137
+61396535373630343037376537396431373362643639393633636433326335353230366161656362
-38353862663730343234333566643936356562383632313238303166646438646435623765373362
+63313933393235386664353761613530636332366332383134353936313639306435356462616639
-66323339656466653235346661353266383339616364613562656233653935653739323262353661
+62386564363766306334346637353166376361353634366331326638643735373038626333666361
-35356338363035373066323238323364336438643839313435313163383935316163396335303231
+61623163356532373765633530316635313161346434626538333332613233316630366565346534
-36303133636539316661396664376639653265376266366432326633323734313165356537656337
+62336436333838303732366536626433353135636362333436613763323730396562616361306665
-61633835303735366332336134613733336534646531393265633437373862316262663066393262
+35646634623861396232626533333265343761393632393161363063646663663938363535353531
-61646663363239633430363165346534386639383562316161363532396266613837346230323663
+34636433353237386362313132633732646438643230653438313761386335333731393337346665
-33623539633637666362346332323833316165643436353332363038343436666536336461636130
+39316239626636323435303932613637373231623337353838313337356632336234623434623038
-37383839393866386139343565373164626639326530666662323230373030333938393531326435
+66366435376434366364353737656230393531633636633036333630376133313165333963636432
-61306436623362373363623135336139343162393236326463666664323465646436366561323331
+32353431666532373436316133353439383461353834346439313531333338333764316264343136
-30396663643765396234346265353831623634343963393234306532613336353732373630363830
+32353733363031376337336666636537613032376361343533323362626132396632633533643163
-31613561353464306363316136383463396361353933313239643732353335656232636230323539
+66313862623433636438613230646338653961343861623433623864326163363135633864373231
-64316163316461666564353637626532363966313332353362383936643661363066353734666631
+66313935353164363466356164616363653761623565663032313264656565623864383732376334
-62363562613362333436313534326135393665663930376535646562646635326236363163626632
+31613538623166663736373535363633623937323261386433386436373361623162626361363033
-31376334336265323737326138373532323363393937303635373663653862393730646532616637
+35393063663664373230613635353762333238353937633730623861626236663935333134326132
-34643235636165343063633836623936666564313566303861356332636130393635353438613637
+61343864376639633164333436623563633635343236333664333663653431643664386631376162
-64303430653061356533373235336661363139643537633337386164303236613934313566643431
+39613766393530313938653562333630343765316461326665386664643134643661666539373131
-65393664333233326565653634656566393738366566613137383436366638656561376135626364
+35373565313763336136653035656138313162333965663565353531336362616637363830383462
-38303633343737633464356134616331366266613164386439346338373036666337386632376638
+62343866623838343066653035613031346362303263636436656434303039393434643531666238
-62316566646539633961353865636165313966663339336436316165323966326561363166613134
+31633363373036356336333235363134616362393362636561316265363366386530666465656531
-32373764333839313338353162326363373430393031333038646631333836323237643537376462
+37366431373564656533363534613633393739663666666566303538363139643833323537356163
-33623836396536343335333665366561363737333864363963383836353234633739626466316561
+61396533353536333330343130326663613135393237653438323439623836363162393435646236
-63346638316365363364316530656563343537326534353137396433646333626666313735366331
+36636631366234663536323463303538303434633632316438343935353162316632663939313437
-31373465303032306636373437393366316639393065336336306130346234313038316539353037
+36666538323463643462323234626262333131353238333031346139333535656539363336646332
-36333164306566313539633464373132643234306335633361386637393231306566333832386566
+30353830623536396662313264323637663637353934636532306331323166316535343131336639
-35356661633535306531623961346635613730653566663536393234373839613961626632313837
+32396237313539653030366164343336623463656261616661376638346561646632623434393166
-62363062346534623961373266363561326666316161643366386133323163636532363437623266
+62383033313931653235356236363862393837616365616332653730383833376165323735333632
-38646464366463353162376635313764353338616439633566633862636238643265663465396161
+33303966643462626438303132383233663065353032643362306331663632616535346362643137
-65333238623833346631653264336430656539623561353135353363326139323234376333346436
+33323736393038356362356135363733326263303430633136383137653734363331623331373537
-31633365613730663133656532653937373334386335643138663666626230343339663232656336
+63353833336236626664616265383464633335623861353739623863653866323534343163393466
-36613931623233303164646630363966353730643531356130643265363332386333313132343433
+37666163383465383734643430386437613866616361393561336364346437346164313665363634
-37653233336337373533313839393365623532376439656537326439663864326639636462613830
+32303539613165613631353239666339336639303561303234336135326137613363656335353761
-38323832333865613139336632363534616639313566303131326339353934396534336261333839
+37616537353132353561303730326330386435636165303464616232633531613132623636653432
-63303730363732613037386265663132326264613435666138633639303761623361623836616163
+34353637336338626564353364613962393365333639653133356165343032326430616237396536
-62663263376231383036663062376333656362303666383962333762653066396339393231636533
+63653033326238336363353061303031393064616163656162376362663061643236643232333266
-37386538636635366463663434653564656664316230653836646639333736316434356339393435
+62653761383338323837383361383965323963393935626634333661356661396139356566303830
-39656564333330393436336135656262363862353263613664643063633365336161366664353765
+38313133313564353030643866313366646338376666396435356264373239636666373861363964
-36356232613234386265396436346130353763636538346636663234633237663133323066316563
+31363863393033633063326237666630666631393036656233336238353736343534633238393532
-31636237643538376632663462626363386234306334303062343530306161306265633031366161
+62663335393839613137373863346263396361386235346439323437353531626537313965663262
-63393830656333633864376335623231653230396635616331666236666661643330356135343931
+32636434386238323634616336336464333963633432333932653462666661393933666531303136
-35356335323332346361666538343065643565333133393137323536363438326563313531336336
+34363432386637323136656335306663656232626631663464396565303465323636326431343762
-39613330653331356436326437653936386531663037336539643165316131663435363766326435
+66383339336133636431353538643838663331373736636563626537623361363231633934663931
-37316466666166303262383265653833633437313732363632636235363037326561353032623134
+35366365333036366661363263393062373130383062646332636330326139343266666234323835
-6239663434363939386230356530333036656637303161626465
+31636463633237373532363333306136396437356236303961623133353630653435396462313264
+34336239373839663061346461313137393333306534646465366430393164646430613964323638
+62666638346130383464633339396364643835323036303039656230343564623663313238326333
+30653364613661306539373832616638636563653963353835343265383865306233356438303464
+62303761363839316237653036316563303466373763323164316331356263656664393831396130
+32636135306166366230353834313330383035383964353031663431613434623331616165613565
+34623765663564636463363431643736613433316366393862353433323032616435303334396230
+38356266623566356637373561343331366665373964373564616138306531356439

group_vars/linode.yaml

@@ -19,8 +19,9 @@
 #  - 2600:3c00::c
 
 dns_servers:
-  - 127.0.0.1
+  - 8.8.8.8
-  - ::1
+  - 1.1.1.1
+  - 9.9.9.9
 
 timezone: Etc/UTC
 

group_vars/monitor_servers/main.yaml

@@ -8,6 +8,8 @@ alertmanager_web_external_url: https://monitor.kill0.net/alertmanager
 prometheus_web_route_prefix: /
 alertmanager_web_route_prefix: /
 
+prometheus_file_sd_config_d_files: []
+
 prometheus_config:
   global:
     scrape_interval: 15s
@@ -16,6 +18,10 @@ prometheus_config:
       region: dallas
       provider: linode
       replica: A
+  remote_write:
+    - url: http://localhost:9009/api/v1/push
+      headers:
+        X-Scope-OrgID: kill0-net
   alerting:
     alertmanagers:
       - static_configs:
@@ -75,10 +81,13 @@ prometheus_config:
             - dns.google
             - vpn-home.kill0.net
             - ping-home.kill0.net
-            - 10.255.0.16
+            - 169.254.0.2
             - vpn1-sch.corp.nmi.com
-            - vpn-chi.ops.nmi.com
+            - gp-chi.ops.nmi.com
-            - vpn-ash.ops.nmi.com
+            - gp-ash.ops.nmi.com
+            - 172.16.100.1
+            - 172.16.100.2
+            - 172.16.10.16
       relabel_configs:
         - source_labels: [__address__]
           target_label: __param_target
@@ -174,6 +183,77 @@ prometheus_config:
       static_configs:
         - targets:
             - "localhost:3002"
+#    - job_name: process-exporter
+#      scrape_interval: 5s
+#      static_configs:
+#        - targets:
+#            - "localhost:9256"
+    - job_name: loki
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - "localhost:3100"
+    - job_name: promtail
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - jump0.kill0.net:9080
+            - mine0.kill0.net:9080
+    - job_name: gitea
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:3001
+    - job_name: karma
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:8080
+    - job_name: kthxbye
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:8081
+    - job_name: smokeping
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:9374
+    - job_name: mimir
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:9009
+    - &snmp_job
+      job_name: snmp
+      static_configs:
+        - targets:
+          - 172.16.100.1
+          - 172.16.100.2
+      metrics_path: /snmp
+      params:
+        auth: [public_v2]
+        module:
+          - if_mib
+          - ip_mib
+      relabel_configs:
+        - source_labels: [__address__]
+          target_label: __param_target
+        - source_labels: [__param_target]
+          target_label: instance
+        - target_label: __address__
+          replacement: 127.0.0.1:9116
+    - job_name: snmp_exporter
+      static_configs:
+        - targets:
+          - localhost:9116
+    - <<: *snmp_job
+      job_name: snmp-long
+      scrape_interval: 30s
+      scrape_timeout: 30s
+      static_configs:
+        - targets: []
+
   rule_files:
     - rules.yaml
 
@@ -225,6 +305,10 @@ prometheus_rules_config:
           expr: up{job=~"thanos.+"} == 0
           labels:
             severity: critical
+        - alert: Down
+          expr: up == 0
+          labels:
+            severity: critical
         - alert: FileSystemUsage
           expr: ((node_filesystem_size_bytes{mountpoint!~"fuse.lxcfs|tmpfs"} - node_filesystem_free_bytes) / node_filesystem_size_bytes) > 0.80
           for: 1m
@@ -277,6 +361,13 @@ prometheus_rules_config:
             # summary: Certificates expiring in < 14 days
             summary: "{% raw %}Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}){% endraw %}"
             description: "{% raw %}SSL certificate expires in 14 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}{% endraw %}"
+    - name: snmp.rules
+      rules:
+        - alert: PortDown
+          expr: ifAdminStatus{ifName=~"(Gi|eth).+", ifAlias!~".+laptop|notebook.+"} == 1 and ifOperStatus == 2
+          for: 1m
+        - alert: PortFlapping
+          expr: changes(ifOperStatus{ifName=~"(Gi|eth).+"}[5m]) > 2
 
 blackbox_exporter_config:
   modules:
@@ -306,34 +397,6 @@ blackbox_exporter_config:
       http:
         method: GET
 
-#  route:
-#    receiver: pushover-receiver
-#    mute_time_intervals:
-#      - quiet_hours
-#    routes:
-#      - receiver: blackhole
-#        match:
-#          alertname: MaintenanceMode
-#      #- receiver: blackhole
-#      #  match:
-#      #    alertname: QuietHours
-#  receivers:
-#    - name: blackhole
-#    - name: pushover-receiver
-#      pushover_configs:
-#        - token: "{{ vault_pushover_token }}"
-#          user_key: "{{ vault_pushover_user_key }}"
-#  inhibit_rules:
-#    - source_match:
-#        alertname: MaintenanceMode
-#    #- source_match:
-#    #    alertname: QuietHours
-#  time_intervals:
-#    - name: quiet_hours
-#      times:
-#        - start_time: 03:00
-#          end_time: 15:00
-
 alertmanager_config:
   inhibit_rules:
     - source_match:
@@ -342,9 +405,13 @@ alertmanager_config:
     - name: blackhole
     - name: pushover-receiver
       pushover_configs:
-        - token: agwd6wv7xveakykb8e5rz7rw3eg2v3
+        - token: "{{ vault_alertmanager_pushover_token }}"
           user_key: 28G1x3lT4oUtlck50R1H3e6j8kDHjb
+    - name: discord
+      discord_configs:
+        - webhook_url: "{{ vault_alertmanager_discord_webhook_url }}"
   route:
+    repeat_interval: 24h
     receiver: pushover-receiver
     routes:
       - match:
@@ -356,6 +423,8 @@ alertmanager_config:
       - receiver: pushover-receiver
         mute_time_intervals:
           - quiet_hours
+        continue: true
+      - receiver: discord
   time_intervals:
     - name: quiet_hours
       time_intervals:
@@ -371,7 +440,7 @@ node_exporter_du_directories:
   - /var/lib/loki
 
 firewall_ipset_loki:
-  - 10.255.0.0/24
+  - 169.254.0.0/24
 
 karma_config:
   alertmanager:
@@ -416,3 +485,112 @@ karma_config:
 thanos_bucket_config: "{{ vault_thanos_bucket_config }}"
 
 kthxbye_listen: :8081
+
+loki_storage_config:
+  tsdb_shipper:
+    active_index_directory: "{{ loki_var_path }}/tsdb-index"
+    cache_location: "{{ loki_var_path }}/tsdb-cache"
+  gcs:
+    bucket_name: kill0-net-loki
+    service_account: "{{ vault_loki_gcs_service_account | string }}"
+
+loki_schema_config:
+  configs:
+    - from: 2023-08-11
+      index:
+        period: 24h
+        prefix: index_
+      object_store: gcs
+      schema: v12
+      store: tsdb
+    - from: 2024-04-10
+      index:
+        period: 24h
+        prefix: index_
+      object_store: gcs
+      schema: v13
+      store: tsdb
+
+loki_query_scheduler:
+  max_outstanding_requests_per_tenant: 32768
+
+loki_querier:
+  max_concurrent: 16
+
+loki_compactor:
+  working_directory: "{{ loki_var_path }}/retention"
+  delete_request_store: gcs
+  compaction_interval: 10m
+  retention_enabled: true
+  retention_delete_delay: 2h
+  retention_delete_worker_count: 150
+
+loki_ruler:
+  alertmanager_url: http://localhost:9093
+  storage:
+    type: gcs
+    gcs:
+      bucket_name: kill0-net-loki
+      service_account: "{{ vault_loki_gcs_service_account | string }}"
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true
+
+rsyslog_d:
+  - name: loki
+    priority: 10
+    content: |
+      if $hostname == [ "ap0", "coresw0", "fw0", "power0", "172.16.100.1", "172.16.100.2" ] then {
+        action(
+          type="omfwd"
+          target="localhost"
+          port="1514"
+          protocol="tcp"
+          action.resumeretrycount="-1"
+          queue.type="linkedlist"
+          queue.size="1000000"
+          queue.filename="loki-fwd"
+          queue.saveonshutdown="on"
+          keepalive="on"
+          template="RSYSLOG_SyslogProtocol23Format"
+          tcp_framing="octet-counted"
+        )
+      }
+
+smokeping_prober_config:
+  targets:
+    - hosts:
+      - dns.google
+      - vpn-home.kill0.net
+      - ping-home.kill0.net
+      - vpn1-sch.corp.nmi.com
+      - gp-chi.ops.nmi.com
+      - gp-ash.ops.nmi.com
+      - 169.254.0.2
+      - 172.16.100.1
+      - 172.16.100.2
+      - 172.16.10.16
+      network: ip4
+    - hosts:
+      - dns.google
+      - ping-home.kill0.net
+      - fc00::ffff:169.255.0.2
+      - fc00::ffff:169.255.0.16
+      network: ip6
+
+mimir_common:
+  storage:
+    backend: gcs
+    gcs:
+      bucket_name: kill0-net-mimir
+      service_account: "{{ vault_mimir_gcs_service_account | string }}"
+
+mimir_blocks_storage:
+  storage_prefix: blocks
+
+mimir_alertmanager_storage:
+  storage_prefix: alertmanager
+
+mimir_ruler_storage:
+  storage_prefix: ruler

group_vars/monitor_servers/vault.yaml

@@ -0,0 +1,17 @@
+$ANSIBLE_VAULT;1.1;AES256
+35346264373635663161356339313438613932623165613239353162316265333231623434383030
+6435323137313638633663356635373464393730663834320a346362633362323864373636346165
+37363637663037653932313165653333643833376133383336363930623338333134623562353239
+6430363062323865650a363330653031383666386637633333646339393064396330313037363239
+30626538373432633031666264646236613936333965366430653031303131626161376633346435
+63323165366666663362353661353634636339393930343862336132613466636131343861343835
+64633531336139353961626565363434316230393739626531366661653132616566363234393036
+35656331383038396665376236373531323931313632656331356235353664636264393664346131
+38633038303364373166366633646330393636366134626437376662386235626233633831363062
+32636461646661613734353739663934333365313932306363666464656236366634653032303031
+34333032373935343366626537386231306666663934326664353432323338353235306231363464
+64653561663662363064313436653036613038633033623737666335636331656461653535643864
+62376539343761666366333331373164623230663639373231373763653938343535646166303639
+31616463316364366130653033643935356461363938386264306162623933336338363365316162
+63396436316338306136616265643562353931356239393661333161396537653366643765303031
+64323639653263323837

group_vars/stats_servers/main.yaml

@@ -24,9 +24,8 @@ grafana_config:
     http_port: "{{ grafana_port }}"
 
 grafana_ssl_enabled: true
-grafana_ssl_certificate: "/etc/letsencrypt/live/{{ grafana_domain }}/fullchain.pem"
+grafana_ssl_certificate: "/var/lib/lego/certificates/{{ grafana_domain }}.crt"
-grafana_ssl_certificate_key: "/etc/letsencrypt/live/{{ grafana_domain }}/privkey.pem"
+grafana_ssl_certificate_key: "/var/lib/lego/certificates/{{ grafana_domain }}.key"
-# grafana_ssl_dhparam: "/etc/letsencrypt/ssl-dhparams.pem"
 
 grafana_datasources:
   apiVersion: 1

host_vars/jump0.kill0.net/all.yaml

@@ -18,38 +18,126 @@ certbot_certificates:
   - domains:
       - cavi.cc
     email: rcavicchioni@gmail.com
+  - domains:
+      - proxy.kill0.net
+    email: rcavicchioni@gmail.com
+
+lego_user_environ:
+  GCE_PROJECT: kill0-net
+  GCE_SERVICE_ACCOUNT_FILE: "{{ lego_etc_dir_path }}/credentials.json"
+
+lego_bin_user_args:
+  - --email rcavicchioni@gmail.com
+  - --dns gcloud
+
+lego_bin_renew_user_args:
+  - --renew-hook "systemctl reload nginx"
+
+lego_domains:
+  - name: cavi.cc
+  - name: dl.kill0.net
+  - name: git.kill0.net
+  - name: monitor.kill0.net
+  - name: proxy.kill0.net
+  - name: stats.kill0.net
 
 autossh_config: []
 
 wireguard_interfaces:
   wg0:
-    address: 10.255.0.1/32
+    address:
+      - 169.254.0.1/24
+      - fc00::ffff:169.254.0.1/64
     private_key: "{{ vault_wireguard_private_keys.wg0 }}"
     listen_port: 51820
+    table: 'off'
   wg1:
     address:
       - 192.168.255.1/24
-      - 2600:3c00:e000:343::1/128
+      - fc01::ffff:192.168.255.1/128
+      - 2600:3c00:e000:343::ffff:192.168.255.1/128
     private_key: "{{ vault_wireguard_private_keys.wg1 }}"
     listen_port: 51821
 
 restic_tidy_enabled: true
 
+nginx_htpasswd_files: "{{ vault_nginx_htpasswd_files }}"
+
 nginx_vhosts:
   cavicc:
-    - server_name: cavi.cc
+    server:
-      root: /var/www/cavicc
+      - server_name: cavi.cc
-      listen:
+        root: /var/www/cavicc
-        - 80
+        listen:
-        - "[::]:80"
+          - 80
-      raw: |
+          - "[::]:80"
-        location / {
+        raw: |
-            return 301 https://$server_name$request_uri;
+          location / {
-        }
+              return 301 https://$server_name$request_uri;
-    - server_name: cavi.cc
+          }
-      root: /var/www/cavicc
+      - server_name: cavi.cc
-      listen:
+        root: /var/www/cavicc
-        - 443 ssl http2
+        listen:
-        - "[::]:443 ssl http2"
+          - 443 ssl
-      ssl_certificate: /etc/letsencrypt/live/cavi.cc/fullchain.pem
+          - "[::]:443 ssl"
-      ssl_certificate_key: /etc/letsencrypt/live/cavi.cc/privkey.pem
+        ssl_certificate: /var/lib/lego/certificates/cavi.cc.crt
+        ssl_certificate_key: /var/lib/lego/certificates/cavi.cc.key
+        # ssl_certificate: /etc/letsencrypt/live/cavi.cc/fullchain.pem
+        # ssl_certificate_key: /etc/letsencrypt/live/cavi.cc/privkey.pem
+        raw: |
+          location / {
+            add_header Alt-Svc 'h3=":$server_port"; ma=86400'; 
+          }
+
+  proxy:
+    upstream:
+      - name: loki_backend
+        server:
+          - localhost:3100
+      #- name: prometheus_backend
+      #  server:
+      #    - localhost:9090
+    map:
+      - name: $http_upgrade
+        variable: $connection_upgrade
+        content:
+          default: upgrade
+          '': close
+    server:
+      - server_name: proxy.kill0.net
+        root: /var/empty
+        listen:
+          - 80
+          - "[::]:80"
+        raw: |
+          location / {
+              return 301 https://$server_name$request_uri;
+          }
+      - server_name: proxy.kill0.net
+        root: /var/empty
+        listen:
+          - 443 ssl
+          - "[::]:443 ssl"
+        # ssl_certificate: /etc/letsencrypt/live/proxy.kill0.net/fullchain.pem
+        # ssl_certificate_key: /etc/letsencrypt/live/proxy.kill0.net/privkey.pem
+        ssl_certificate: /var/lib/lego/certificates/proxy.kill0.net.crt
+        ssl_certificate_key: /var/lib/lego/certificates/proxy.kill0.net.key
+        raw: |
+          auth_basic "Proxy";
+          auth_basic_user_file /etc/nginx/proxy.htpasswd;
+
+          location / {
+            add_header Alt-Svc 'h3=":$server_port"; ma=86400'; 
+          }
+
+          location /loki {
+            proxy_http_version 1.1;
+            proxy_pass http://loki_backend;
+            proxy_set_header Connection $connection_upgrade;
+            proxy_set_header Host $http_host;
+            proxy_set_header Upgrade $http_upgrade;
+          }
+
+          location /prometheus/ {
+            proxy_pass http://prometheus_backend/;
+          }

host_vars/mine0.kill0.net/main.yaml

@@ -161,6 +161,8 @@ openvpn_certificates:
 
 wireguard_interfaces:
   wg0:
-    address: 10.255.0.16/32
+    address:
+      - 169.254.0.2/24
+      - fc00::ffff:169.254.0.2/64
     private_key: "{{ vault_wireguard_private_keys.wg0 }}"
     listen_port: 51820

host_vars/nal-hutta.yaml

@@ -1,22 +0,0 @@
----
-#network_interfaces:
-#  - name: eth0
-#    address:
-#      - 45.56.123.101/24
-#      - 2600:3c00::f03c:91ff:fed5:eeec/64
-#    gateway:
-#      - 45.56.123.1
-#      - fe80::1
-
-firewall_allowed_tcp_ports:
-  v4:
-    - 443
-    - 80
-    - 8186
-  v6:
-    - 443
-    - 80
-    - 8186
-
-postfix_sasl_passwd_map:
-  "[smtp.fastmail.com]:465": "foo:bar"

host_vars/rmq1.yaml

@@ -1,17 +0,0 @@
----
-keepalived_vrrp_instances:
-  VI_1:
-    state: MASTER
-    interface: eth0
-    virtual_router_id: 51
-    priority: 254
-    authentication:
-      auth_type: PASS
-      auth_pass: asdf
-    unicast_peer: |
-      {{ groups['rabbitmq_servers'] | map('extract', hostvars, ['ansible_eth0', 'ipv4', 'address']) | difference([ansible_default_ipv4.address])| list }}
-    virtual_ipaddress:
-      - 10.100.100.20/24
-    track_script:
-      - chk_rabbitmq
-      - chk_amqp_port

host_vars/rmq2.yaml

@@ -1,17 +0,0 @@
----
-keepalived_vrrp_instances:
-  VI_1:
-    state: BACKUP
-    interface: eth0
-    virtual_router_id: 51
-    priority: 253
-    authentication:
-      auth_type: PASS
-      auth_pass: asdf
-    unicast_peer: |
-      {{ groups['rabbitmq_servers'] | map('extract', hostvars, ['ansible_eth0', 'ipv4', 'address']) | difference([ansible_default_ipv4.address])| list }}
-    virtual_ipaddress:
-      - 10.100.100.20/24
-    track_script:
-      - chk_rabbitmq
-      - chk_amqp_port

host_vars/rmq3.yaml

@@ -1,17 +0,0 @@
----
-keepalived_vrrp_instances:
-  VI_1:
-    state: BACKUP
-    interface: eth0
-    virtual_router_id: 51
-    priority: 252
-    authentication:
-      auth_type: PASS
-      auth_pass: asdf
-    unicast_peer: |
-      {{ groups['rabbitmq_servers'] | map('extract', hostvars, ['ansible_eth0', 'ipv4', 'address']) | difference([ansible_default_ipv4.address])| list }}
-    virtual_ipaddress:
-      - 10.100.100.20/24
-    track_script:
-      - chk_rabbitmq
-      - chk_amqp_port

host_vars/ubuntu.yaml

@@ -1,7 +0,0 @@
----
-#network_interfaces:
-#  - name: enp1s0
-#    address:
-#      - 192.168.124.124/24
-#    gateway4: 192.168.124.1
-#

playbook.yaml

@@ -3,25 +3,59 @@
   become: true
   roles:
     - common
-    - network
+    - role: network
+      tags:
+        - network
+        - netplan
     - util
     - sudo
     - hostsfile
     - certs
-    - rsyslog
+    - role: rsyslog
+      tags:
+        - rsyslog
+        - syslog
+        - logging
     - users
     - dns
-    - firewall
+    - role: firewall
+      tags:
+        - firewall
+        - iptables
     - openssh
-    - wireguard
+    - role: wireguard
+      tags:
+        - wireguard
+        - vpn
     - chrony
     - unattended-upgrades
     - postfix
     - restic
-    - node_exporter
+    - role: node_exporter
-    - blackbox_exporter
+      tags:
-    - mtail
+        - prometheus
+        - monitoring
+    - role: blackbox_exporter
+      tags:
+        - prometheus
+        - monitoring
+    - role: mtail
+      tags:
+        - prometheus
+        - monitoring
     - supervisor
+    # - vector
+    - role: promtail
+      tags:
+        - promtail
+        - loki
+        - logging
+    - role: cloudflared
+      tags:
+        - cloudflared
+        - zerotrust
+        - access
+        - vpn
 - hosts: minecraft_servers
   become: true
   roles:
@@ -34,35 +68,98 @@
 - hosts: git_servers
   become: true
   roles:
-    - nginx
+    - role: certbot
-    - certbot
+      tags:
-    - gitea
+        - tls
+    - role: nginx
+      tags:
+        - nginx
+    - role: gitea
+      tags:
+        - gitea
+        - git
 - hosts: stats_servers
   become: true
   roles:
-    - nginx
+    - role: certbot
-    - certbot
+      tags:
-    - grafana
+        - tls
+    - role: nginx
+      tags:
+        - nginx
+    - role: grafana
+      tags:
+        - grafana
+        - monitoring
+        - o11y
 - hosts: monitor_servers
   become: true
   roles:
-    - nginx
+    - certbot
+    - role: nginx
+      tags:
+        - nginx
     - role: prometheus
       tags:
         - prometheus
         - monitoring
-    - alertmanager
+    - role: alertmanager
-    - blackbox_exporter
+      tags:
-    - pushgateway
+        - prometheus
+        - monitoring
+    - role: blackbox_exporter
+      tags:
+        - prometheus
+        - monitoring
+    - role: pushgateway
+      tags:
+        - prometheus
+        - monitoring
     - role: karma
       tags:
+        - prometheus
         - monitoring
     - role: kthxbye
       tags:
+        - prometheus
         - monitoring
     - role: thanos
       tags:
+        - prometheus
         - thanos
         - monitoring
+    - role: loki
+      tags:
+        - loki
+        - logging
+    - role: logcli
+      tags:
+        - logcli
+        - loki
+        - logging
+    - role: smokeping_prober
+      tags:
+        - prometheus
+        - monitoring
+        - smokeping
+    - role: mimir
+      tags:
+        - prometheus
+        - mimir
+        - monitoring
+    - role: snmp_exporter
+      tags:
+        - prometheus
+        - snmp_exporter
+        - monitoring
+    - role: lego
+      tags:
+        - acme
+        - certificates
+        - lego
+        - letsencrypt
+        - pki
+        - tls
+
 
 # vim:ft=yaml.ansible:

roles/certbot/defaults/main.yaml

@@ -1,22 +1,35 @@
 ---
 certbot_package_name: certbot
-certbot_package_state: present
+certbot_package_state: latest
+
+certbot_plugins:
+  - certbot-dns-cloudflare
+  - certbot-dns-digitalocean
+  - certbot-dns-dnsimple
+  - certbot-dns-dnsmadeeasy
+  - certbot-dns-gehirn
+  - certbot-dns-google
+  - certbot-dns-linode
+  - certbot-dns-luadns
+  - certbot-dns-nsone
+  - certbot-dns-ovh
+  - certbot-dns-rfc2136
+  - certbot-dns-route53
+  - certbot-dns-sakuracloud
 
 certbot_service_name: certbot.service
 
+certbot_bin_path: /usr/local/bin
+certbot_path: "{{ certbot_bin_path }}/certbot"
+
 certbot_timer_name: certbot.timer
 certbot_timer_state: started
-certbot_timer_enabled: yes
+certbot_timer_enabled: true
 
-certbot_cron_state: present
+certbot_etc_path: /etc/letsencrypt
-certbot_cron_user: root
+certbot_live_path: "{{ certbot_etc_path }}/live"
-certbot_cron_file_path: /etc/cron.d/certbot
-certbot_cron_env:
-  path: /usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-  shell: /bin/sh
-certbot_cron_command: test -x /usr/bin/certbot -a \! -d /run/systemd/system && perl -e 'sleep int(rand(43200))' && certbot -q renew
-certbot_cron_hour: "*/12"
-certbot_cron_minute: "0"
 
 certbot_system_timer_on_calender: "*-*-* 00,12:00:00"
 certbot_system_timer_randomized_delay_sec: 43200
+
+certbot_credential_path: /root/.secrets/certbot

roles/certbot/handlers/main.yaml

@@ -1,6 +1,4 @@
 ---
 - name: systemd daemon-reload
-  systemd:
+  ansible.builtin.systemd:
-    name: "{{ certbot_service_name }}"
+    daemon_reload: true
-    daemon_reload: yes
-    state: restarted

roles/certbot/tasks/configure-linode.yaml

@@ -0,0 +1,23 @@
+---
+- name: configure linode credentials
+  ansible.builtin.copy:
+    dest: "{{ certbot_credential_path }}/linode.ini"
+    owner: root
+    group: root
+    mode: 0600
+    content: "{{ certbot_dns_linode_credentials }}"
+  no_log: true
+
+- name: certbot (linode)
+  ansible.builtin.shell: >
+    certbot certonly \
+      --dns-linode \
+      --dns-linode-credentials "{{ certbot_credential_path }}/linode.ini" \
+      --quiet \
+      --agree-tos \
+      --noninteractive \
+      --email "{{ item.email }}" \
+      --domain "{{ item.domains | join(',') }}"
+  args:
+    creates: "{{ certbot_live_path }}/{{ item.domains | first }}/cert.pem"
+  loop: "{{ certbot_certificates | default([]) }}"

roles/certbot/tasks/issue.yaml

@@ -1,9 +1 @@
 ---
-- name: "determine if certificate for {{ item.domains | join(', ') }}" 
-  stat:
-    path: "/etc/letsencrypt/live/{{ item.domains | first }}/cert.pem"
-  register: st
-
-- name: "request certificate for {{ item.domains | join(', ') }}"
-  command: "certbot certonly -q --webroot -w {{ certbot_challenge_webroot_path }} --agree-tos --noninteractive --email {{ item.email }} -d {{ item.domains | join(',') }}"
-  when: not st.stat.exists

roles/certbot/tasks/main.yaml

@@ -23,65 +23,51 @@
       paths:
         - tasks
 
-- name: install certbot modules
+- name: install certbot
-  package:
+  ansible.builtin.pip:
     name: "{{ certbot_package_name }}"
     state: "{{ certbot_package_state }}"
 
-- name: configure challenge webroot
+- name: install certbot plugins
-  file:
+  ansible.builtin.pip:
-    path: "{{ certbot_challenge_webroot_path }}"
+    name: "{{ certbot_plugins }}"
-    state: "directory"
+    state: latest
+
+- name: create credential path
+  ansible.builtin.file:
+    path: "{{ certbot_credential_path }}"
     owner: root
     group: root
-    mode: 0755
+    mode: 0700
+    state: directory
 
 - name: request certificates
   ansible.builtin.include_tasks: "issue.yaml"
   loop: "{{ certbot_certificates }}"
 
-- name: configure systemd timer
+- name: include linode tasks
-  block:
+  ansible.builtin.include_tasks: configure-linode.yaml
-    - name: create systemd timer override directory
-      file:
-        path: "/etc/systemd/system/{{ certbot_timer_name }}.d"
-        owner: root
-        group: root
-        mode: 0755
-        state: directory
 
-    - name: configure systemd timer options
+- name: configure renewal service
-      template:
+  ansible.builtin.template:
-        src: certbot.timer.j2
+    src: certbot.service.j2
-        dest: "/etc/systemd/system/{{ certbot_timer_name }}.d/override.conf"
+    dest: "/etc/systemd/system/certbot.service"
-        owner: root
+    owner: root
-        group: root
+    group: root
-        mode: 0644
+    mode: 0644
-      notify: systemd daemon-reload
+  notify: systemd daemon-reload
-    - name: enable the timer
-      systemd:
-        name: "{{ certbot_timer_name }}"
-        state: "{{ certbot_timer_state }}"
-        enabled: "{{ certbot_timer_enabled }}"
-  when: ansible_service_mgr == "systemd"
 
-- name: configure cron job
+- name: configure renewal timer
-  block:
+  ansible.builtin.template:
-    - name: configure env
+    src: certbot.timer.j2
-      cron:
+    dest: "/etc/systemd/system/certbot.timer"
-        name: "{{ item.key | upper }}"
+    owner: root
-        env: yes
+    group: root
-        job: "{{ item.value }}"
+    mode: 0644
-        user: "{{ certbot_cron_user }}"
+  notify: systemd daemon-reload
-        cron_file: "{{ certbot_cron_file_path }}"
+
-        state: "{{ certbot_cron_state }}"
+- name: manage timer
-      loop: "{{ certbot_cron_env | dict2items }}"
+  ansible.builtin.systemd:
-    - name: create job
+    name: "{{ certbot_timer_name }}"
-      cron:
+    enabled: "{{ certbot_timer_enabled }}"
-        name: certbot
+    state: "{{ certbot_timer_state }}"
-        user: "{{ certbot_cron_user }}"
-        hour: "{{ certbot_cron_hour }}"
-        minute: "{{ certbot_cron_minute }}"
-        cron_file: "{{ certbot_cron_file_path }}"
-        job: "{{ certbot_cron_command }}"
-        state: "{{ certbot_cron_state }}"

roles/certbot/templates/certbot.service.j2

@@ -0,0 +1,14 @@
+# {{ ansible_managed }}
+
+[Unit]
+Description=Certbot renewal
+After=network-online.target
+Wants=network-online.target
+Wants={{ certbot_timer_name }}
+
+[Service]
+Type=oneshot
+ExecStart={{ certbot_path }} --quiet renew
+
+[Install]
+WantedBy=multi-user.target

roles/certbot/templates/certbot.timer.j2

@@ -1,5 +1,12 @@
 # {{ ansible_managed }}
 
+[Unit]
+Description=Certbot renewal
+Requires={{ certbot_service_name }}
+
 [Timer]
 OnCalendar={{ certbot_system_timer_on_calender }} 
 RandomizedDelaySec={{ certbot_system_timer_randomized_delay_sec }}
+
+[Install]
+WantedBy=timers.target

roles/cloudflared/defaults/main.yaml

@@ -0,0 +1,10 @@
+---
+cloudflared_package_name: cloudflared
+cloudflared_package_state: present
+
+cloudflared_service_name: cloudflared.service
+cloudflared_service_enabled: true
+cloudflared_service_state: started
+
+cloudflared_apt_repository_repo: "deb [signed-by=/etc/apt/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared {{ ansible_lsb.codename }} main"
+cloudflared_apt_repository_state: present

roles/cloudflared/tasks/Debian.yaml

@@ -0,0 +1,14 @@
+---
+- name: trust cloudflare apt respository key
+  ansible.builtin.copy:
+    src: "cloudflare-main.gpg"
+    dest: "/etc/apt/keyrings/cloudflare-main.gpg"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: configure cloudflare apt repository
+  ansible.builtin.apt_repository:
+    repo: "{{ cloudflared_apt_repository_repo }}"
+    state: "{{ cloudflared_apt_repository_state | default('present') }}"
+    filename: cloudflared

roles/cloudflared/tasks/install.yaml

@@ -0,0 +1,5 @@
+---
+- name: install package
+  ansible.builtin.package:
+    name: "{{ cloudflared_package_name }}"
+    state: "{{ cloudflared_package_state | default('present') }}"

roles/cloudflared/tasks/main.yaml

@@ -0,0 +1,28 @@
+---
+- name: gather os specific variables
+  ansible.builtin.include_vars: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: include os specific tasks
+  ansible.builtin.include_tasks: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- ansible.builtin.include_tasks: install.yaml
+
+# - ansible.builtin.include_tasks: configure.yaml

roles/dl/defaults/main.yaml

@@ -4,5 +4,5 @@ dl_server_root: /var/www/dl
 dl_access_log: /var/log/nginx/dl.access.log
 dl_error_log: /var/log/nginx/dl.error.log
 dl_ssl_enabled: false
-dl_ssl_certificate: "/etc/letsencrypt/live/{{ dl_server_name }}/fullchain.pem"
+dl_ssl_certificate: "/var/lib/lego/certificates/{{ dl_server_name }}.crt"
-dl_ssl_certificate_key: "/etc/letsencrypt/live/{{ dl_server_name }}/privkey.pem"
+dl_ssl_certificate_key: "/var/lib/lego/certificates/{{ dl_server_name }}.key"

roles/dl/templates/nginx.conf.j2

@@ -26,10 +26,13 @@ server {
 {% if dl_ssl_enabled is defined and
       dl_ssl_enabled %}
 server {
-    listen 443 ssl http2;
+    listen 443 ssl;
 {% if ansible_all_ipv6_addresses | length %}
-    listen [::]:443 ssl http2;
+    listen [::]:443 ssl;
 {% endif %}
+
+    http2 on;
+
     server_name {{ dl_server_name }};
     access_log {{ dl_access_log }} main;
     error_log {{ dl_error_log }} warn;
@@ -46,6 +49,10 @@ server {
     ssl_dhparam {{ dl_ssl_dhparam }};
 {% endif %}
 
+    location / {
+        add_header Alt-Svc 'h3=":$server_port"; ma=86400';
+    }
+
     location ~ ^\/~(.+?)(\/.*)?$ {
         alias /home/$1/public_html$2;
         index index.html index.htm;

roles/docker/handlers/main.yaml

@@ -0,0 +1,12 @@
+---
+- name: reload docker
+  ansible.builtin.service:
+    name: "{{ docker_service_name | default('docker') }}"
+    state: reloaded
+
+- name: restart docker
+  ansible.builtin.service:
+    name: "{{ docker_service_name | default('docker') }}"
+    state: restarted
+  listen:
+    - restart nftables

roles/firewall/templates/ip6tables.j2

@@ -130,6 +130,9 @@
 {% endif %}
 {% if firewall_ipset_syslog is defined %}
 -A INPUT -p tcp -m tcp --dport 514 -m set --match-set syslog6 src -m comment --comment "accept syslog 514/tcp6" -j LOG_ACCEPT
+-A INPUT -p udp -m udp --dport 514 -m set --match-set syslog6 src -m comment --comment "accept syslog 514/udp6" -j LOG_ACCEPT
+-A INPUT -p tcp -m tcp --dport 1514 -m set --match-set syslog6 src -m comment --comment "accept syslog 1514/tcp6" -j LOG_ACCEPT
+-A INPUT -p udp -m udp --dport 1514 -m set --match-set syslog6 src -m comment --comment "accept syslog 1514/udp6" -j LOG_ACCEPT
 {% endif %}
 {% if firewall_ipset_influxdb is defined %}
 -A INPUT -p tcp -m tcp --dport 8086 -m set --match-set influxdb6 src -m comment --comment "accept influxdb 8086/tcp6" -j LOG_ACCEPT

roles/firewall/templates/iptables.j2

@@ -117,6 +117,8 @@
 {% if firewall_ipset_syslog is defined %}
 -A INPUT -p tcp -m tcp --dport 514 -m set --match-set syslog4 src -m comment --comment "accept syslog 514/tcp" -j LOG_ACCEPT
 -A INPUT -p udp -m udp --dport 514 -m set --match-set syslog4 src -m comment --comment "accept syslog 514/udp" -j LOG_ACCEPT
+-A INPUT -p tcp -m tcp --dport 1514 -m set --match-set syslog4 src -m comment --comment "accept syslog 1514/tcp" -j LOG_ACCEPT
+-A INPUT -p udp -m udp --dport 1514 -m set --match-set syslog4 src -m comment --comment "accept syslog 1514/udp" -j LOG_ACCEPT
 {% endif %}
 {% if firewall_ipset_influxdb is defined %}
 -A INPUT -p tcp -m tcp --dport 8086 -m set --match-set influxdb4 src -m comment --comment "accept influxdb 8086/tcp" -j LOG_ACCEPT

roles/gitea/defaults/main.yaml

@@ -53,6 +53,8 @@ gitea_config:
     colorize: no
   service:
     register_manual_confirm: true
+  metrics:
+    enabled: true
 
 gitea_var_tree:
   - "{{ gitea_var_path }}"
@@ -62,6 +64,6 @@ gitea_var_tree:
   - "{{ gitea_var_path }}/backup"
 
 gitea_ssl_enabled: yes
-gitea_ssl_certificate: "/etc/letsencrypt/live/{{ gitea_domain }}/fullchain.pem"
+gitea_ssl_certificate: "/var/lib/lego/certificates/{{ gitea_domain }}.crt"
-gitea_ssl_certificate_key: "/etc/letsencrypt/live/{{ gitea_domain }}/privkey.pem"
+gitea_ssl_certificate_key: "/var/lib/lego/certificates/{{ gitea_domain }}.key"
 #gitea_ssl_dhparam: "/etc/letsencrypt/ssl-dhparams.pem"

roles/gitea/templates/nginx.conf.j2

@@ -37,10 +37,13 @@ server {
 {% if gitea_ssl_enabled is defined and
       gitea_ssl_enabled %}
 server {
-    listen 443 ssl http2;
+    listen 443 ssl;
 {% if ansible_all_ipv6_addresses | length %}
-    listen [::]:443 ssl http2;
+    listen [::]:443 ssl;
 {% endif %}
+
+    http2 on;
+
     server_name {{ gitea_domain }};
 
     access_log /var/log/nginx/gitea.access.log main;
@@ -62,6 +65,7 @@ server {
     }
 
     location / {
+        add_header Alt-Svc 'h3=":$server_port"; ma=86400';
         limit_req zone=req_bad_actors burst=10 nodelay;
         proxy_pass http://gitea_backend;
     }

roles/grafana/defaults/main.yaml

@@ -26,8 +26,8 @@ grafana_config:
     http_port: "{{ grafana_port }}"
 
 grafana_ssl_enabled: true
-grafana_ssl_certificate: "/etc/letsencrypt/live/{{ grafana_domain }}/fullchain.pem"
+grafana_ssl_certificate: "/var/lib/lego/certificates/{{ grafana_domain }}.crt"
-grafana_ssl_certificate_key: "/etc/letsencrypt/live/{{ grafana_domain }}/privkey.pem"
+grafana_ssl_certificate_key: "/var/lib/lego/certificates/{{ grafana_domain }}.key"
 # grafana_ssl_dhparam: "/etc/letsencrypt/ssl-dhparams.pem"
 
 

roles/grafana/templates/nginx.conf.j2

@@ -6,6 +6,11 @@ upstream grafana_backend {
     server 127.0.0.1:{{ grafana_port }};
 }
 
+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    ''      close;
+}
+
 server {
     listen 80;
 {% if ansible_all_ipv6_addresses | length %}
@@ -32,10 +37,13 @@ server {
 {% if grafana_ssl_enabled is defined and
       grafana_ssl_enabled %}
 server {
-    listen 443 ssl http2;
+    listen 443 ssl;
 {% if ansible_all_ipv6_addresses | length %}
-    listen [::]:443 ssl http2;
+    listen [::]:443 ssl;
 {% endif %}
+
+    http2 on;
+
     server_name {{ grafana_domain }};
 
     access_log /var/log/nginx/grafana.access.log main;
@@ -59,7 +67,12 @@ server {
     }
 
     location / {
+        add_header Alt-Svc 'h3=":$server_port"; ma=86400';
         limit_req zone=req_bad_actors burst=10 nodelay;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+        proxy_set_header Host $http_host;
         proxy_pass http://grafana_backend;
     }
 }

roles/loki/defaults/main.yaml

@@ -26,12 +26,17 @@ loki_user_shell: /usr/sbin/nologin
 loki_group: loki
 loki_group_state: "{{ loki_user_state | default('present') }}"
 
-loki_config_path: /etc/loki.yaml
-
 loki_var_path: /var/lib/loki
 loki_var_owner: "{{ loki_user }}"
 loki_var_group: "{{ loki_group }}"
-loki_var_mode: "0755"
+loki_var_mode: "0700"
+
+loki_etc_path: /etc/loki
+loki_etc_owner: "{{ loki_user }}"
+loki_etc_group: "{{ loki_group }}"
+loki_etc_mode: "0755"
+
+loki_config_path: "{{ loki_etc_path }}/config.yaml"
 
 loki_bin_path: /usr/local/bin
 
@@ -39,36 +44,51 @@ loki_auth_enabled: false
 
 loki_server:
   http_listen_port: 3100
+  grpc_listen_port: 9096
 
-loki_ingester:
+loki_common:
-  lifecycler:
+  instance_addr: 127.0.0.1
-    address: 127.0.0.1
+  path_prefix: "{{ loki_var_path }}"
-    ring:
+  storage:
-      kvstore:
+    filesystem:
-        store: inmemory
+      chunks_directory: "{{ loki_var_path }}/chunks"
-      replication_factor: 1
+      rules_directory: "{{ loki_var_path }}/rules"
-    final_sleep: 0s
+  replication_factor: 1
-  chunk_idle_period: 5m
+  ring:
-  chunk_retain_period: 30s
+    kvstore:
+      store: inmemory
+
+loki_query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+# loki_storage_config:
+#   {}
 
 loki_schema_config:
   configs:
-  - from: 2020-05-15
+  - from: 2020-10-24
-    store: boltdb
+    store: boltdb-shipper
-    object_store: filesystem
+    object_store: gcs
     schema: v11
     index:
       prefix: index_
-      period: 168h
+      period: 24h
 
-loki_storage_config:
+loki_ruler:
-  boltdb:
+  alertmanager_url: http://localhost:9093
-    directory: "{{ loki_var_path }}/index"
+
-  filesystem:
+# loki_query_scheduler:
-    directory: "{{ loki_var_path }}/chunks"
+#   {}
+
+# loki_querier:
+#   {}
+
+# loki_compactor:
+#   {}
 
 loki_limits_config:
-  enforce_metric_name: false
+  retention_period: 744h
-  reject_old_samples: true
-  reject_old_samples_max_age: 168h
-  ingestion_burst_size_mb: 16

roles/loki/handlers/main.yaml

@@ -3,4 +3,4 @@
   systemd:
     name: loki.service
     daemon_reload: true
-    state: restarted
+    state: restarted

roles/loki/tasks/configure.yaml

@@ -15,14 +15,13 @@
     home: "{{ loki_var_path }}"
     state: "{{ loki_user_state | default('present') }}"
 
-- name: configure
+- name: create etc path
-  template:
+  file:
-    src: loki.yaml.j2
+    path: "{{ loki_etc_path }}"
-    dest: "{{ loki_config_path }}"
+    state: directory
-    owner: root
+    owner: "{{ loki_etc_owner }}"
-    group: root
+    group: "{{ loki_etc_group }}"
-    mode: 0444
+    mode: "{{ loki_etc_mode }}"
-  notify: restart loki
 
 - name: create var path
   file:
@@ -32,6 +31,15 @@
     group: "{{ loki_var_group }}"
     mode: "{{ loki_var_mode }}"
 
+- name: configure
+  template:
+    src: config.yaml.j2
+    dest: "{{ loki_config_path }}"
+    owner: "{{ loki_user }}"
+    group: "{{ loki_group }}"
+    mode: 0400
+  notify: restart loki
+
 - name: configure systemd template
   template:
     src: "{{ loki_service_name }}.j2"

roles/loki/templates/config.yaml.j2

@@ -0,0 +1,55 @@
+{{ ansible_managed | comment }}
+---
+{% if loki_auth_enabled is defined %}
+auth_enabled: {{ loki_auth_enabled | bool | lower }}
+{% endif %}
+
+{% if loki_server is defined %}
+server:
+  {{ loki_server | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_common is defined %}
+common:
+  {{ loki_common | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_query_range is defined %}
+query_range:
+  {{ loki_query_range | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_storage_config is defined %}
+storage_config:
+  {{ loki_storage_config | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_schema_config is defined %}
+schema_config:
+  {{ loki_schema_config | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_ruler is defined %}
+ruler:
+  {{ loki_ruler | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_query_scheduler is defined %}
+query_scheduler:
+  {{ loki_query_scheduler | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_querier is defined %}
+querier:
+  {{ loki_querier | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_compactor is defined %}
+compactor:
+  {{ loki_compactor | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_limits_config is defined %}
+limits_config:
+  {{ loki_limits_config | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}

roles/loki/templates/loki.service.j2

@@ -1,19 +1,19 @@
 {{ ansible_managed | comment }}
 
 [Unit]
-Description=Loki
+Description=Loki service
-After=network-online.target
+After=network.target
 
 [Service]
 Type=simple
 User={{ loki_user }}
-Group={{ loki_group }}
 ExecStart={{ loki_bin_path }}/loki \
   -config.file {{ loki_config_path }}
-WorkingDirectory={{ loki_var_path }}
 
-Restart=always
+WorkingDirectory={{ loki_var_path }}
-RestartSec=1
+TimeoutSec = 120
+Restart = on-failure
+RestartSec = 2
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target

roles/loki/templates/loki.yaml.j2

@@ -1,30 +0,0 @@
-{{ ansible_managed | comment }}
----
-{% if loki_auth_enabled is defined %}
-auth_enabled: {{ loki_auth_enabled | bool | lower }}
-{% endif %}
-
-{% if loki_server is defined %}
-server:
-  {{ loki_server | to_nice_yaml(indent=2) | indent(2, False) }}
-{% endif -%}
-
-{% if loki_ingester is defined %}
-ingester:
-  {{ loki_ingester | to_nice_yaml(indent=2) | indent(2, False) }}
-{% endif -%}
-
-{% if loki_schema_config is defined %}
-schema_config:
-  {{ loki_schema_config | to_nice_yaml(indent=2) | indent(2, False) }}
-{% endif -%}
-
-{% if loki_storage_config is defined %}
-storage_config:
-  {{ loki_storage_config | to_nice_yaml(indent=2) | indent(2, False) }}
-{% endif -%}
-
-{% if loki_limits_config is defined %}
-limits_config:
-  {{ loki_limits_config | to_nice_yaml(indent=2) | indent(2, False) }}
-{% endif -%}

roles/minecraft/defaults/main.yaml

@@ -11,8 +11,8 @@ minecraft_port: 25565
 minecraft_user: minecraft
 minecraft_group: minecraft
 
-minecraft_jar_url: https://launcher.mojang.com/v1/objects/e00c4052dac1d59a1188b2aa9d5a87113aaf1122/server.jar
+minecraft_jar_url: https://piston-data.mojang.com/v1/objects/84194a2f286ef7c14ed7ce0090dba59902951553/server.jar
-minecraft_jar_checksum: sha256:deefd056f0cf89c3d7fd48d03f56a8a73943586e8c061fdabd0fd92d32ced2b2
+minecraft_jar_checksum: sha256:3af73a9dc5a102e38147946360dd27d4d70bae7055bf91cf2151cd5d121b79e0
 
 minecraft_opt_path: /opt/minecraft
 minecraft_var_path: /var/opt/minecraft

roles/mtail/defaults/main.yaml

@@ -12,7 +12,7 @@ mtail_service_enabled: yes
 mtail_version_regex: ^mtail version (\S+)
 
 mtail_github_project_url: https://github.com/google/mtail
-mtail_release_file: "mtail_{{ mtail_version }}_{{ ansible_system | capitalize }}_{{ ansible_architecture }}.tar.gz"
+mtail_release_file: "mtail_{{ mtail_version }}_{{ ansible_system | lower }}_{{ mtail_go_arch }}.tar.gz"
 mtail_release_url: "{{ mtail_github_project_url }}/releases/download/v{{ mtail_version }}/{{ mtail_release_file }}"
 mtail_download_path: "/tmp/{{ mtail_release_file }}"
 mtail_checksum_url: "{{ mtail_github_project_url }}/releases/download/v{{ mtail_version }}/checksums.txt"

roles/mtail/tasks/pre.yaml

@@ -1,42 +1,4 @@
 ---
-#- name: determine if installed
-#  stat:
-#    path: "{{ mtail_bin_path }}/mtail"
-#  register: st
-#
-#- name: set mtail_installed
-#  set_fact:
-#    mtail_installed: "{{ st.stat.exists | bool }}"
-#
-#- block:
-#  - name: determine latest version
-#    uri:
-#      url: https://api.github.com/repos/google/mtail/releases/latest
-#      return_content: true
-#      body_format: json
-#    register: _latest_version
-#    until: _latest_version.status == 200
-#    retries: 3
-#
-#  - name: set mtail_version
-#    set_fact:
-#      mtail_version: "{{ _latest_version.json['tag_name'] | regex_replace('^v', '') }}"
-#
-#- block:
-#  - name: determine installed version
-#    command: "{{ mtail_bin_path }}/mtail --version"
-#    register: _installed_version_string
-#    changed_when: false
-#
-#  - name: set mtail_local_version
-#    set_fact:
-#      mtail_local_version: "{{ _installed_version_string.stdout | regex_search(mtail_version_regex, '\\1') | first }}"
-#  when: mtail_installed
-#
-#- name: set mtail_local_version to 0
-#  set_fact:
-#    mtail_local_version: "0"
-#  when: not mtail_installed
 - name: determine if installed
   stat:
     path: "{{ mtail_bin_path }}/mtail"

roles/network/defaults/main.yml

@@ -6,6 +6,23 @@ network_netplan_config_path: "{{ network_netplan_etc_path }}/ansible.yaml"
 network_netplan_default_config_path: "{{ network_netplan_etc_path }}/01-netcfg.yaml"
 # network_netplan_default_config_state: absent
 
+network_netplan:
+  network:
+    version: 2
+    ethernets:
+      eth0:
+        dhcp4: false
+        dhcp6: false
+        accept-ra: true
+        addresses:
+          - "{{ ansible_default_ipv4.address }}/{{ (ansible_default_ipv4.network + '/' + ansible_default_ipv4.netmask) | ansible.utils.ipaddr('prefix') }}"
+          - "{{ ansible_default_ipv6.address }}/{{ ansible_default_ipv6.prefix }}"
+        routes:
+          - to: default
+            via: "{{ ansible_default_ipv4.gateway }}"
+        nameservers:
+          addresses: "{{ network_dns_nameservers }}"
+
 network_interfaces:
   - name: eth0
     inet4:
@@ -15,6 +32,7 @@ network_interfaces:
       gateway: "{{ ansible_default_ipv4.gateway }}"
     inet6:
       dhcp: false
+      accept_ra: true
       address:
         - "{{ ansible_default_ipv6.address }}/{{ ansible_default_ipv6.prefix }}"
       gateway: "{{ ansible_default_ipv6.gateway }}"

roles/network/tasks/netplan.yml

@@ -5,14 +5,14 @@
     state: "{{ network_netplan_default_config_state | default('absent') }}"
     owner: root
     group: root
-    mode: 0644
+    mode: '0400'
   notify: netplan apply
 
 - name: Configure netplan
-  ansible.builtin.template:
+  ansible.builtin.copy:
     dest: "{{ network_netplan_config_path }}"
-    src: netplan.yaml.j2
+    content: "{{ network_netplan | to_nice_yaml }}"
     owner: root
     group: root
-    mode: '0644'
+    mode: '0400'
   notify: netplan apply

roles/network/templates/netplan.yaml.j2

@@ -1,16 +1,19 @@
 ---
 network:
-  version: "{{ network_netplan_version | default(2) }}"
+  version: {{ network_netplan_version | default(2) }}
-  renderer: "{{ network_netplan_renderer | default("networkd") }}"
+  renderer: {{ network_netplan_renderer | default('networkd') }}
 {% if network_interfaces is defined and network_interfaces | length %}
   ethernets:
 {%   for iface in network_interfaces %}
     {{ iface['name'] }}:
 {%     if iface['inet4']['dhcp'] is defined %}
-      dhcp4: "{{ iface['inet4']['dhcp'] | ternary('yes', 'no') }}"
+      dhcp4: {{ iface['inet4']['dhcp'] | ternary('true', 'false') }}
 {%     endif %}
 {%     if iface['inet4']['dhcp'] is defined %}
-      dhcp6: "{{ iface['inet6']['dhcp'] | ternary('yes', 'no') }}"
+      dhcp6: {{ iface['inet6']['dhcp'] | ternary('true', 'false') }}
+{%     endif %}
+{%     if iface['inet6']['accept_ra'] is defined %}
+      accept-ra: {{ iface['inet6']['accept_ra'] | ternary('true', 'false') }}
 {%     endif %}
 {%     if iface['inet4']['address'] is defined or iface['inet6']['address'] is defined %}
       addresses:
@@ -22,10 +25,10 @@ network:
 {%       endfor %}
 {%     endif %}
 {%     if iface['inet4']['gateway'] is defined %}
-      gateway4: "{{ iface['inet4']['gateway'] }}"
+      gateway4: {{ iface['inet4']['gateway'] }}
 {%     endif %}
 {%     if iface['inet6']['gateway'] is defined %}
-      gateway6: "{{ iface['inet6']['gateway'] }}"
+      gateway6: {{ iface['inet6']['gateway'] }}
 {%     endif %}
 {%     if network_dns_nameservers is defined %}
       nameservers:

roles/nftables/defaults/main.yaml

@@ -36,35 +36,54 @@ nftables_builtin_sets:
     - flags interval
 
 nftables_input_builtin_rules:
-  - type filter hook input priority filter; policy drop;
+  '000 policy':
-  - ip saddr @blackhole4 drop
+    - type filter hook input priority filter; policy drop;
-  - ip6 saddr @blackhole6 drop
+  '010 blackhole':
-  - ct state established,related accept
+    - ip saddr @blackhole4 drop
-  - ct state invalid drop
+    - ip6 saddr @blackhole6 drop
-  - iifname "lo" accept
+  '020 related established':
-  - icmpv6 type $REQUIRED_ICMPV6_TYPES accept
+    - ct state established,related accept
-  - icmpv6 type echo-request accept
+    - ct state invalid drop
-  - icmp type echo-request accept
+  '030 loopback':
-  - tcp dport @tcp_input_accept accept
+    - iifname "lo" accept
-  - udp dport @udp_input_accept accept
+  '040 icmp':
-  # this should be last because these ports could be allowed
+    - icmpv6 type $REQUIRED_ICMPV6_TYPES accept
-  - udp dport $TRACEROUTE_UDP_PORTS reject
+    - icmpv6 type echo-request accept
+    - icmp type echo-request accept
+  '050 tcp accept':
+    - tcp dport @tcp_input_accept accept
+  '060 udp accept':
+    - udp dport @udp_input_accept accept
+  '999 traceroute':
+    # this should be last because these ports could be allowed
+    - udp dport $TRACEROUTE_UDP_PORTS reject
 
 nftables_forward_builtin_rules:
-  - type filter hook forward priority filter; policy drop;
+  '000 policy':
-  - ct state { established, related } accept
+    - type filter hook forward priority filter; policy drop;
+  '010 related established':
+    - ct state { established, related } accept
 
 nftables_output_builtin_rules:
-  - type filter hook output priority filter; policy accept;
+  '000 policy':
-  - ip daddr @blackhole4 drop
+    - type filter hook output priority filter; policy accept;
-  - ip6 daddr @blackhole6 drop
+  '010 blackhole':
-  - ct state { established, related } accept
+    - ip daddr @blackhole4 drop
+    - ip6 daddr @blackhole6 drop
+  '020 related established':
+    - ct state { established, related } accept
 
-# nftables_sets:
+nftables_defines:
-#   {}
+  {} 
-#
+
-# nftables_input_rules:
+nftables_sets:
-#   []
+  {}
-#
+
-# nftables_output_rules:
+nftables_input_rules:
-#   []
+  {}
+
+nftables_forward_rules:
+  {}
+
+nftables_output_rules:
+  {}

roles/nftables/templates/nftables.conf.j2

@@ -1,82 +1,53 @@
+{% set combined_defines = [ nftables_builtin_defines, nftables_defines ] | combine %}
+{% set combined_sets = [ nftables_builtin_sets, nftables_sets ] | combine %}
+{% set combined_input_rules = [ nftables_input_builtin_rules, nftables_input_rules ] | combine %}
+{% set combined_forward_rules = [ nftables_forward_builtin_rules, nftables_forward_rules ] | combine %}
+{% set combined_output_rules = [ nftables_output_builtin_rules, nftables_output_rules ] | combine %}
 table inet filter {
-{% if nftables_builtin_defines is mapping %}
+{% for name, cfg in combined_defines.items() %}
-{%   for name, cfg in nftables_builtin_defines.items() %}
+{%   if cfg is string or cfg is number %}
-{%     if cfg is string %}
   define {{ name }} = {{ cfg }}
-{%     elif cfg is sequence %} 
+{%  elif cfg is sequence %} 
   define {{ name }} = {
-{%     for elem in cfg %}
+{%   for elem in cfg %}
     {{ elem }},
-{%     endfor %}
-  }
-{%     endif %}
 {%   endfor %}
-{% endif %}
-{% if nftables_defines is mapping %}
-{%   for name, cfg in nftables_defines.items() %}
-  define {{ name }} = {
-{%     for elem in cfg %}
-    {{ elem }},
-{%     endfor %}
   }
-{%   endfor %}
+{%   endif %}
-{% endif %}
+{% endfor %}
 
-{% if nftables_builtin_sets is mapping %}
+{% for name, cfg in combined_sets.items() %}
-{%   for name, cfg in nftables_builtin_sets.items() %}
   set {{ name }} { 
-{%     for elem in cfg %}
+{%   for elem in cfg %}
     {{ elem }}
-{%     endfor %}
-  }
 {%   endfor %}
-{% endif %}
-{% if nftables_sets is mapping %}
-{%   for name, cfg in nftables_sets.items() %}
-  set {{ name }} { 
-{%     for elem in cfg %}
-    {{ elem }}
-{%     endfor %}
   }
-{%   endfor %}
+{% endfor %}
-{% endif %}
 
   chain input {
-{% if nftables_input_builtin_rules is sequence %}
+{% for comment, rules in combined_input_rules.items() %}
-{%   for rule in nftables_input_builtin_rules %}
+    # {{ comment }}
+{%   for rule in rules %}
     {{ rule }}
 {%   endfor %}
-{% endif %}
+{% endfor %}
-{% if nftables_input_rules is sequence %}
-{%   for rule in nftables_input_rules %}
-    {{ rule }}
-{%   endfor %}
-{% endif %}
   }
 
   chain forward {
-{% if nftables_forward_builtin_rules is sequence %}
+{% for comment, rules in combined_forward_rules.items() %}
-{%   for rule in nftables_forward_builtin_rules %}
+    # {{ comment }}
+{%   for rule in rules %}
     {{ rule }}
 {%   endfor %}
-{% endif %}
+{% endfor %}
-{% if nftables_forward_rules is sequence %}
-{%   for rule in nftables_forward_rules %}
-    {{ rule }}
-{%   endfor %}
-{% endif %}
   }
 
   chain output {
-{% if nftables_output_builtin_rules is sequence %}
+{% for comment, rules in combined_output_rules.items() %}
-{%   for rule in nftables_output_builtin_rules %}
+    # {{ comment }}
+{%   for rule in rules %}
     {{ rule }}
 {%   endfor %}
-{% endif %}
+{% endfor %}
-{% if nftables_output_rules is sequence %}
-{%   for rule in nftables_output_rules %}
-    {{ rule }}
-{%   endfor %}
-{% endif %}
   }
 }

roles/nginx/tasks/main.yml

@@ -44,6 +44,19 @@
     mode: 0644
   notify: reload nginx
 
+- name: configure htpasswd files
+  ansible.builtin.copy:
+    dest: "{{ nginx_etc_path }}/{{ item.key }}.htpasswd"
+    owner: root
+    group: nginx
+    mode: 0640
+    content: |
+      {% for u, h in item.value.items() %}
+      {{ u }}:{{ h }}
+      {% endfor %}
+  loop: "{{ nginx_htpasswd_files | dict2items }}"
+  notify: reload nginx
+
 - name: configure virtual hosts
   ansible.builtin.include_tasks: vhost.yaml
   loop: "{{ nginx_vhosts | dict2items }}"
@@ -52,4 +65,4 @@
   service:
     name: "{{ nginx_service_name }}"
     state: "{{ nginx_service_state }}"
-    enabled: "{{ nginx_service_enabled }}"
+    enabled: "{{ nginx_service_enabled }}"

roles/nginx/tasks/vhost.yaml

@@ -3,11 +3,11 @@
   block:
   - name: create webroot
     file:
-      path: "{{ vhost.root }}"
+      path: "{{ server.root }}"
       state: directory
-    loop: "{{ item.value }}"
+    loop: "{{ item.value.server }}"
     loop_control:
-      loop_var: vhost
+      loop_var: server
 
   - name: configure virtual host
     template:

roles/nginx/templates/vhost.conf.j2

@@ -1,33 +1,59 @@
 # {{ ansible_managed }}
 
-{% for vhost in item.value %}
+{% if item.value.upstream is defined %}
+{% for upstream in item.value.upstream %}
+upstream {{ upstream.name }} {
+{% for server in upstream.server %}
+    server {{ server }};
+{% endfor %}
+}
+{% endfor %}
+{% endif %}
+
+{% if item.value.map is defined %}
+{% for map in item.value.map %}
+map {{ map.name }} {{ map.variable }} {
+{% for k, v in map.content.items() %}
+{% if k is string and k == "" %}
+    "" {{ v }};
+{% else %}
+    {{ k }} {{ v }};
+{% endif %}
+{% endfor %}
+}
+{% endfor %}
+{% endif %}
+
+{% for server in item.value.server %}
 server {
-{% if vhost.listen is defined %}
+{% if server.listen is defined %}
-{% for listen in vhost.listen %}
+{% for listen in server.listen %}
     listen {{ listen }};
 {% endfor %}
-
-{% if vhost.server_name is defined %}
-    server_name {{ vhost.server_name }};
-{% endif %}
-{% endif %}
-    access_log {{ vhost.access_log | default(nginx_var_log_path + '/' + vhost.server_name + '.access.log main') }};
-    error_log {{ vhost.error_log | default(nginx_var_log_path + '/' + vhost.server_name + '.error.log warn') }};
-
-{% if vhost.root is defined %}
-    root {{ vhost.root }};
 {% endif %}
 
-    index {{ vhost.index | default('index.html index.htm') }};
+    http2 {{ server.http2 | default("on") }};
 
-{% if vhost.ssl_certificate is defined %}
+{% if server.server_name is defined %}
-    ssl_certificate {{ vhost.ssl_certificate }};
+    server_name {{ server.server_name }};
 {% endif %}
-{% if vhost.ssl_certificate_key is defined %}
+    access_log {{ server.access_log | default(nginx_var_log_path + '/' + server.server_name + '.access.log main') }};
-    ssl_certificate_key {{ vhost.ssl_certificate_key }};
+    error_log {{ server.error_log | default(nginx_var_log_path + '/' + server.server_name + '.error.log warn') }};
+
+{% if server.root is defined %}
+    root {{ server.root }};
 {% endif %}
-{% if vhost.ssl_dhparam is defined %}
+
-    ssl_dhparam {{ vhost.ssl_dhparam }};
+    index {{ server.index | default('index.html index.htm') }};
+
+{% if server.ssl_certificate is defined %}
+    ssl_certificate {{ server.ssl_certificate }};
+{% endif %}
+{% if server.ssl_certificate_key is defined %}
+    ssl_certificate_key {{ server.ssl_certificate_key }};
+{% endif %}
+{% if server.ssl_dhparam is defined %}
+    ssl_dhparam {{ server.ssl_dhparam }};
 {% endif %}
 
     location /.well-known/acme-challenge/ {
@@ -35,8 +61,8 @@ server {
         try_files $uri =404;
     }
 
-{% if vhost.raw is defined %}
+{% if server.raw is defined %}
-    {{ vhost.raw | indent(4) }}
+    {{ server.raw | indent(4) }}
 {% endif %}
 }
-{% endfor %}
+{% endfor %}

roles/prometheus/defaults/main.yaml

@@ -39,8 +39,8 @@ prometheus_bin_path: /usr/local/bin
 
 prometheus_ssl_enabled: true
 prometheus_hostname: "{{ prometheus_web_external_url | urlsplit('hostname') }}"
-prometheus_ssl_certificate: "/etc/letsencrypt/live/{{ prometheus_hostname }}/fullchain.pem"
+prometheus_ssl_certificate: "/var/lib/lego/certificates/{{ prometheus_hostname }}.crt"
-prometheus_ssl_certificate_key: "/etc/letsencrypt/live/{{ prometheus_hostname }}/privkey.pem"
+prometheus_ssl_certificate_key: "/var/lib/lego/certificates/{{ prometheus_hostname }}.key"
 
 prometheus_alertmanager_enabled: true
 

roles/prometheus/tasks/configure.yaml

@@ -23,6 +23,14 @@
     group: "{{ prometheus_etc_group }}"
     mode: "{{ prometheus_etc_mode }}"
 
+- name: create file_sd_config.d path
+  file:
+    path: "{{ prometheus_etc_path }}/file_sd_config.d"
+    state: directory
+    owner: "{{ prometheus_etc_owner }}"
+    group: "{{ prometheus_etc_group }}"
+    mode: "{{ prometheus_etc_mode }}"
+
 - name: create var path
   file:
     path: "{{ prometheus_var_path }}"
@@ -49,6 +57,15 @@
     mode: 0444
   notify: reload prometheus
 
+- name: configure file_sd_config.d
+  copy:
+    dest: "{{ prometheus_etc_path }}/file_sd_config.d/{{ item.name }}"
+    content: "{{ (item.targets | default([])) | to_json }}"
+    owner: root
+    group: root
+    mode: 0444
+  loop: "{{ prometheus_file_sd_config_d_files | default([]) }}"
+
 - name: configure systemd template
   template:
     src: prometheus.service.j2

roles/prometheus/templates/nginx.conf.j2

@@ -38,10 +38,13 @@ server {
 {% if prometheus_ssl_enabled is defined and
       prometheus_ssl_enabled %}
 server {
-    listen 443 ssl http2;
+    listen 443 ssl;
 {% if ansible_all_ipv6_addresses | length %}
-    listen [::]:443 ssl http2;
+    listen [::]:443 ssl;
 {% endif %}
+
+    http2 on;
+
     server_name {{ prometheus_hostname }};
 
     auth_basic           "Prometheus";
@@ -73,6 +76,7 @@ server {
     }
 
     location / {
+        add_header Alt-Svc 'h3=":$server_port"; ma=86400';
         return 301 /prometheus/;
     }
 }

roles/promtail/templates/promtail.service.j2

@@ -1,19 +1,19 @@
 {{ ansible_managed | comment }}
 
 [Unit]
-Description=Loki
+Description=Promtail service
-After=network-online.target
+After=network.target
 
 [Service]
 Type=simple
 User={{ promtail_user }}
-Group={{ promtail_group }}
 ExecStart={{ promtail_bin_path }}/promtail \
-  -config.file {{ promtail_config_path }}
+  -config.file {{ promtail_config_path }} \
+  -client.external-labels=host=%l
 WorkingDirectory={{ promtail_var_path }}
-
+TimeoutSec = 60
-Restart=always
+Restart=on-failure
-RestartSec=1
+RestartSec=2
 
 [Install]
 WantedBy=multi-user.target

roles/restic/defaults/main.yaml

@@ -1,12 +1,34 @@
 ---
-restic_service_name: restic.service
+restic_go_arch_map:
-restic_service_state: started
+  i386: '386'
-restic_service_enabled: yes
+  x86_64: 'amd64'
+
+
+restic_go_arch: "{{ restic_go_arch_map[ansible_architecture] | default('amd64') }}"
+
+restic_version_regex: ^restic ([\d.]+)
+
+restic_checksum_algo: sha256
+restic_github_rel_path: restic/restic
+restic_github_project_url: "https://github.com/{{ restic_github_rel_path }}"
+restic_release_file: "restic_{{ restic_version }}_{{ ansible_system | lower }}_{{ restic_go_arch }}.bz2"
+restic_release_url: "{{ restic_github_project_url }}/releases/download/v{{ restic_version }}/{{ restic_release_file }}"
+restic_checksum_url: "{{ restic_github_project_url }}/releases/download/v{{ restic_version }}/{{ restic_checksum_algo | upper }}SUMS"
+restic_download_path: "/tmp/{{ restic_release_file }}"
+restic_unarchive_dest_path: /tmp
+restic_extracted_path: "{{ restic_download_path | replace('.bz2', '') }}"
+restic_binaries:
+ - restic
+
+# restic_arch: amd64
+# restic_version: 0.15.2
+# restic_url: "https://github.com/restic/restic/releases/download/v{{ restic_version }}/restic_{{ restic_version }}_linux_{{ restic_arch }}.bz2"
+# restic_checksum: sha256:c8da7350dc334cd5eaf13b2c9d6e689d51e7377ba1784cc6d65977bd44ee1165
+# restic_bin_path: /usr/local/bin
+# restic_etc_path: /etc/restic
+# restic_path: "{{ restic_bin_path }}/restic"
+# restic_self_update: true
 
-restic_arch: amd64
-restic_version: 0.14.0
-restic_url: "https://github.com/restic/restic/releases/download/v{{ restic_version }}/restic_{{ restic_version }}_linux_{{ restic_arch }}.bz2"
-restic_checksum: sha256:c8da7350dc334cd5eaf13b2c9d6e689d51e7377ba1784cc6d65977bd44ee1165
 restic_bin_path: /usr/local/bin
 restic_etc_path: /etc/restic
 restic_path: "{{ restic_bin_path }}/restic"

roles/restic/files/hooks/gitea.sh

@@ -9,7 +9,7 @@ GITEA_CONFIG=${GITEA_CONFIG:-/etc/gitea/app.ini}
 GITEA_WORK_PATH=${GITEA_WORK_PATH:-/var/lib/gitea}
 GITEA_CUSTOM_PATH=${GITEA_CUSTOM_PATH:-$GITEA_WORK_PATH/custom}
 GITEA_BACKUP_PATH=${GITEA_BACKUP_PATH:-$GITEA_WORK_PATH/backup}
-GITEA_KEEP_DAYS=${GITEA_KEEP_DAYS:-2}
+GITEA_KEEP_HOURS=${GITEA_KEEP_HOURS:-12}
 
 prereq() {
     if ! systemctl list-units --full --all | grep -Fq "gitea.service"; then
@@ -41,7 +41,7 @@ main() {
         find "$GITEA_BACKUP_PATH" \
             -type f \
             -name '*.zip' \
-            -mtime "+$GITEA_KEEP_DAYS" \
+            -mmin +$((GITEA_KEEP_HOURS * 60)) \
             -delete
     fi
 }

roles/restic/files/restic-job.sh

@@ -73,6 +73,10 @@ fi
 
 START="$(date +%s)"
 
+if [[ -n "$($RESTIC_PATH list locks -q)" ]]; then
+    error_exit "repo is locked"
+fi
+
 if [ -f "$LOCK" ]; then
     pid=$(cat "$LOCK")
     if ! kill -0 "$pid" 2> /dev/null; then

roles/restic/tasks/install.yaml

@@ -0,0 +1,25 @@
+---
+- block:
+  - name: download
+    get_url:
+      url: "{{ restic_release_url }}"
+      dest: "{{ restic_download_path }}"
+      checksum: "{{ restic_checksum }}"
+    register: dl
+    until: dl is success
+    retries: 5
+    delay: 10
+
+  - name: extract
+    command:
+      cmd: "bunzip2 -f -k {{ restic_download_path }}"
+
+  - name: install binaries
+    copy:
+      src: "{{ restic_extracted_path }}"
+      dest: "{{ restic_path }}"
+      owner: root
+      group: root
+      mode: 0755
+      remote_src: true
+  when: restic_version != restic_local_version

roles/restic/tasks/main.yaml

@@ -23,35 +23,10 @@
       paths:
         - tasks
 
-- name: "download restic {{ restic_version }}"
+- ansible.builtin.include_tasks: pre.yaml
-  get_url:
-    url: "{{ restic_url }}"
-    checksum: "{{ restic_checksum }}"
-    dest: "{{ restic_path }}.bz2"
-    owner: root
-    group: root
-    mode: 0400
-  register: dl
 
-- name: determine if restic exists
+- ansible.builtin.include_tasks: install.yaml
-  stat:
-    path: "{{ restic_path }}"
-  register: st
 
-- name: decompress restic
-  command:
-    cmd: "bunzip2 -k {{ restic_path }}.bz2"
-    creates: "{{ restic_path }}"
-  when: dl.changed or not st.stat.exists
-  #notify:
-  #  - restart restic
-
-- name: manage restic attributes
-  file:
-    path: "{{ restic_path }}"
-    owner: root
-    group: root
-    mode:  0755
 
 - name: create etc tree
   file:

roles/restic/tasks/pre.yaml

@@ -0,0 +1,59 @@
+---
+- name: determine if installed
+  stat:
+    path: "{{ restic_bin_path }}/restic"
+  register: st
+
+- name: set restic_installed
+  set_fact:
+    restic_installed: "{{ st.stat.exists | bool }}"
+
+- block:
+  - name: determine latest version
+    uri:
+      url: "https://api.github.com/repos/{{ restic_github_rel_path }}/releases/latest"
+      return_content: true
+      body_format: json
+    register: _latest_version
+    until: _latest_version.status == 200
+    retries: 3
+
+  - name: set restic_version
+    set_fact:
+      restic_version: "{{ _latest_version.json['tag_name'] | regex_replace('^v', '') }}"
+
+- block:
+  - name: determine installed version
+    command: "{{ restic_bin_path }}/restic version"
+    register: _installed_version_string
+    changed_when: false
+
+  - name: set restic_local_version
+    set_fact:
+      restic_local_version: "{{ _installed_version_string.stdout | regex_search(restic_version_regex, '\\1') | first }}"
+  rescue:
+  - name: set restic_local_version
+    set_fact:
+      restic_local_version: "{{ _installed_version_string.stderr | regex_search(restic_version_regex, '\\1') | first }}"
+  when: restic_installed
+
+- name: set restic_local_version to 0
+  set_fact:
+    restic_local_version: "0"
+  when: not restic_installed
+
+- block:
+  - name: get checksums
+    set_fact:
+      _checksums: "{{ lookup('url', restic_checksum_url, wantlist=True) }}"
+
+  - name: debug
+    debug:
+      msg: "{{ restic_checksum_algo }}:{{ item.split(' ') | first }}"
+    loop: "{{ _checksums }}"
+
+  - name: set restic_checksum
+    set_fact:
+      restic_checksum: "{{ restic_checksum_algo }}:{{ item.split(' ') | first }}"
+    loop: "{{ _checksums }}"
+    when: "restic_release_file in item"

roles/rsyslog/defaults/main.yaml

@@ -33,3 +33,9 @@ rsyslog_default_rules_state: file
 rsyslog_default_rules: []
 rsyslog_rules: []
 rsyslog_archival_format_enabled: false
+
+rsyslog_etc_path: /etc/rsyslog.d
+rsyslog_config_path: /etc/rsyslog.conf
+
+rsyslog_d:
+  []

roles/rsyslog/tasks/archival.yaml

@@ -50,7 +50,7 @@
 - name: configure archival format
   template:
     src: archival.conf.j2
-    dest: /etc/rsyslog.d/10-archival.conf
+    dest: "{{ rsyslog_etc_path }}/10-archival.conf"
     owner: root
     group: root
     mode: 0644
@@ -59,7 +59,7 @@
 
 - name: manage archive rules
   file:
-    path: /etc/rsyslog.d/10-archival.conf
+    path: "{{ rsyslog_etc_path }}/10-archival.conf"
     state: "{{ rsyslog_archival_format_enabled | ternary('file', 'absent') }}"
 
 - name: compress log cron job

roles/rsyslog/tasks/main.yaml

@@ -14,7 +14,7 @@
 - name: configure
   template:
     src: rsyslog.conf.j2
-    dest: /etc/rsyslog.conf
+    dest: "{{ rsyslog_config_path }}"
     owner: root
     group: root
     mode: 0644
@@ -26,7 +26,7 @@
 - name: configure default rules
   template:
     src: default.conf.j2
-    dest: /etc/rsyslog.d/50-default.conf
+    dest: "{{ rsyslog_etc_path }}/50-default.conf"
     owner: root
     group: root
     mode: 0644
@@ -35,8 +35,19 @@
 
 - name: manage default rules
   file:
-    path: /etc/rsyslog.d/50-default.conf
+    path: "{{ rsyslog_etc_path }}/50-default.conf"
     state: "{{ rsyslog_default_rules_state }}"
+  notify: restart rsyslog
+
+- name: configure rsyslog.d rules
+  ansible.builtin.copy:
+    dest: "{{ rsyslog_etc_path }}/{{ item.priority }}-{{ item.name }}.conf"
+    owner: root
+    group: root
+    mode: 0644
+    content: "{{ item.content }}"
+  loop: "{{ rsyslog_d | default([]) }}"
+  notify: restart rsyslog
 
 - name: manage service
   service:

roles/snmp_exporter/defaults/main.yaml

@@ -0,0 +1,102 @@
+---
+snmp_exporter_go_arch_map:
+  i386: '386'
+  x86_64: 'amd64'
+
+snmp_exporter_go_arch: "{{ snmp_exporter_go_arch_map[ansible_architecture] | default('amd64') }}"
+
+snmp_exporter_version: 0.25.0
+snmp_exporter_checksums:
+  snmp_exporter-0.25.0.aix-ppc64.tar.gz: sha256:457524708e136a1c559567eb5170352b25591d33646ad85940f4692b13de8208
+  snmp_exporter-0.25.0.darwin-amd64.tar.gz: sha256:83f820691ec4013614c5e8771c37741ba7732a41f01ac4675428a95cf50785db
+  snmp_exporter-0.25.0.darwin-arm64.tar.gz: sha256:2de16c8ab56c96721ba71ce7b16cdcfaced50f0f7e78fc7ded1747017717a953
+  snmp_exporter-0.25.0.dragonfly-amd64.tar.gz: sha256:a17a8277a134d0f3f5913fdb89b3218e308c01c0749e4b1fe6eff860216c3f06
+  snmp_exporter-0.25.0.freebsd-386.tar.gz: sha256:dc5bb9943ce5abfc4610eb51b98d21754333828acd17e1058f4979dec83ec4bd
+  snmp_exporter-0.25.0.freebsd-amd64.tar.gz: sha256:65c527a32426b781968ee2b1ed9b13542f3333b2f60941ed7261c578d3a19515
+  snmp_exporter-0.25.0.freebsd-arm64.tar.gz: sha256:3ce5dd7c205e148eceef20d4a7f6042b49874d37b2f84cea1ad2b41a7adf27cc
+  snmp_exporter-0.25.0.freebsd-armv6.tar.gz: sha256:fecd7b648de5818f445ee3543b3a0e16090419b83481cb9268f1b070515f4719
+  snmp_exporter-0.25.0.freebsd-armv7.tar.gz: sha256:2750f4d469145a4e9bcf3ae2cf47c3a379581359c224fa3860d88a7671208fe0
+  snmp_exporter-0.25.0.illumos-amd64.tar.gz: sha256:71fbd5973d2b9e06e63728490e820fe5e33f27333a54dcb6b42d152d3cf36d2f
+  snmp_exporter-0.25.0.linux-386.tar.gz: sha256:a78577d5651557a67973363a87db3755170e61a79c8d698f14bc72cde3205e1a
+  snmp_exporter-0.25.0.linux-amd64.tar.gz: sha256:de206a27466656e8b4948ef66dd57cc80c5511ccd285b231fde4e044534db625
+  snmp_exporter-0.25.0.linux-arm64.tar.gz: sha256:d61a38544598921067b546cbdca2cce0165fede0414b2dd769e11b09037164ca
+  snmp_exporter-0.25.0.linux-armv5.tar.gz: sha256:a86cae97116524fc2479bbef211931ca375d78479a276f1c99e4a2ee033d54aa
+  snmp_exporter-0.25.0.linux-armv6.tar.gz: sha256:fed73deb4b2864b9793f07679308117e2b9568e08cf993c640b9fd9a534f2508
+  snmp_exporter-0.25.0.linux-armv7.tar.gz: sha256:ff4ce9ac6f8f489d40d2319ea07428cb58bc6b49ad5cc0054d7475a71b1a68bb
+  snmp_exporter-0.25.0.linux-mips.tar.gz: sha256:616f7d9a798425864852bf8acef1d1fde38e6c85cbc2b6fd176f5bad5aa2ce79
+  snmp_exporter-0.25.0.linux-mips64.tar.gz: sha256:4d7cf894079593e4ae4eba9c10f740514d3defe0ebc362953ffa6ba2ccb93127
+  snmp_exporter-0.25.0.linux-mips64le.tar.gz: sha256:ea3e346a702729daa2a4acb9389cc2fe95549afd6aa5806c173ae0b21340ea0c
+  snmp_exporter-0.25.0.linux-mipsle.tar.gz: sha256:b6fedb56c0ac64b87ec808448ef113bb3a44049d41a70c35004e0e05204a9ba7
+  snmp_exporter-0.25.0.linux-ppc64.tar.gz: sha256:6b6c67ba8e49e1e3e247799f151b74bf1cb6cb65d9e4efcf8c6d0eefa6467dbe
+  snmp_exporter-0.25.0.linux-ppc64le.tar.gz: sha256:b345a5b6808627ca119267f53b4d4835fc831cdbe25922359637b8068b6d2722
+  snmp_exporter-0.25.0.linux-riscv64.tar.gz: sha256:6f3659115b78f05349ce1cc61d17c03e7dbb5830d6a4f13433028efe198e4a66
+  snmp_exporter-0.25.0.linux-s390x.tar.gz: sha256:8a428c63081efee2d15df508c7da5588cc6582a3254561c2ddbd9898520d247e
+  snmp_exporter-0.25.0.netbsd-386.tar.gz: sha256:3b56b8feba1119737fe167db47afb2d53179f03fd1ed2c97a02745486cf78e9d
+  snmp_exporter-0.25.0.netbsd-amd64.tar.gz: sha256:e1e2f82047ec726be64434d45e4d18cff45bf739c8ac7ffcd39d2680148be4f6
+  snmp_exporter-0.25.0.netbsd-arm64.tar.gz: sha256:f1be651984a8aa9fb2793358545da1351cb66c0f94abfa67d97003276aeb64cb
+  snmp_exporter-0.25.0.netbsd-armv6.tar.gz: sha256:d250a3cdd4d6fb572ed740c7f800f2aaa11350294d9275e4054c39bcfed86710
+  snmp_exporter-0.25.0.netbsd-armv7.tar.gz: sha256:0ecc87cc94c6e4f9444e5a508bb3f848753eae551f38715d90531626a09eb21b
+  snmp_exporter-0.25.0.openbsd-386.tar.gz: sha256:93f600e3c8e51c9e4fe2888a6fcac28b6bf4128ff90cf833938c25fcd607d731
+  snmp_exporter-0.25.0.openbsd-amd64.tar.gz: sha256:68b5b7bf8903e02636ea1145a313bad6316950116c7dbcb8e62214acafb76a64
+  snmp_exporter-0.25.0.openbsd-arm64.tar.gz: sha256:ca0ff15972207d7efb0ec08ca3c74ab1940dd780430ebe409214ca6261b4a521
+  snmp_exporter-0.25.0.openbsd-armv7.tar.gz: sha256:094072fcc645e170fbcf617f86f41f35781f6eff83c2a5f3a4327b55c3aae6ba
+  snmp_exporter-0.25.0.windows-386.tar.gz: sha256:feb0eae7fdbff7d96eb489a61e7d4cb6f9065d84e80c5e0f6331893dd3c5e37a
+  snmp_exporter-0.25.0.windows-386.zip: sha256:10cb099383f990303ba293343a98377aabb0575f5d87b8702cd366bd787293b9
+  snmp_exporter-0.25.0.windows-amd64.tar.gz: sha256:78398d2553548f21eaf8920daf86df15865e7c4a93351be01abb10cc2508cc8c
+  snmp_exporter-0.25.0.windows-amd64.zip: sha256:b0872fc2d2cebc60244220c3412185a45b72ac56f2cb36f1e4f35d42e830de2d
+  snmp_exporter-0.25.0.windows-arm64.tar.gz: sha256:e3122f902b714b908884fb10fff61e93960c1ce1a1491d21d7be736ac6c9f833
+  snmp_exporter-0.25.0.windows-arm64.zip: sha256:f3465c09e7a28ced47b15da368074b7df6d610e4c82ea6ae647d916abb541dc8
+
+snmp_exporter_github_rel_path: prometheus/snmp_exporter
+snmp_exporter_github_project_url: "https://github.com/{{ snmp_exporter_github_rel_path }}"
+snmp_exporter_release_file: "snmp_exporter-{{ snmp_exporter_version }}.{{ ansible_system | lower }}-{{ snmp_exporter_go_arch }}.tar.gz"
+snmp_exporter_release_url: "{{ snmp_exporter_github_project_url }}/releases/download/v{{ snmp_exporter_version }}/{{ snmp_exporter_release_file }}"
+snmp_exporter_download_path: "/tmp/{{ snmp_exporter_release_file }}"
+
+snmp_exporter_opt_dir_path: "/opt/snmp_exporter-{{ snmp_exporter_version }}"
+
+snmp_exporter_unarchive_dest_path: /tmp/
+snmp_exporter_extracted_path: "/tmp/{{ snmp_exporter_release_file | replace('.tar.gz', '') }}"
+snmp_exporter_binaries:
+ - snmp_exporter
+
+snmp_exporter_user_name: snmp_exporter
+snmp_exporter_user_shell: /usr/sbin/nologin
+snmp_exporter_user_home: "{{ snmp_exporter_var_dir_path }}"
+snmp_exporter_group_name: snmp_exporter
+
+snmp_exporter_bin_dir_path: /usr/local/bin
+snmp_exporter_bin_path: "{{ snmp_exporter_bin_dir_path }}/snmp_exporter"
+
+snmp_exporter_etc_dir_path: /etc/snmp_exporter
+snmp_exporter_etc_dir_path_owner: "{{ snmp_exporter_user_name }}"
+snmp_exporter_etc_dir_path_group: "{{ snmp_exporter_group_name }}"
+snmp_exporter_etc_dir_path_mode: 0500
+snmp_exporter_etc_dir_path_state: directory
+
+snmp_exporter_var_dir_path: /var/lib/snmp_exporter
+snmp_exporter_var_dir_path_owner: "{{ snmp_exporter_user_name }}"
+snmp_exporter_var_dir_path_group: "{{ snmp_exporter_group_name }}"
+snmp_exporter_var_dir_path_mode: 0500
+snmp_exporter_var_dir_path_state: directory
+
+snmp_exporter_config_file_path: "{{ snmp_exporter_etc_dir_path }}/snmp.yml"
+snmp_exporter_config_file_template_src: snmp.yml.j2
+snmp_exporter_config_file_template_dest: "{{ snmp_exporter_config_file_path }}"
+snmp_exporter_config_file_template_owner: "{{ snmp_exporter_user_name }}"
+snmp_exporter_config_file_template_group: "{{ snmp_exporter_group_name }}"
+snmp_exporter_config_file_template_mode: 0400
+
+snmp_exporter_bin_args:
+  - "--config.file={{ snmp_exporter_config_file_path }}"
+  - "--snmp.module-concurrency={{ ansible_processor_vcpus }}"
+
+snmp_exporter_service_name: snmp_exporter.service
+snmp_exporter_service_enabled: true
+snmp_exporter_service_state: started
+
+snmp_exporter_service_template_src: "{{ snmp_exporter_service_name }}.j2"
+snmp_exporter_service_template_dest: "/etc/systemd/system/{{ snmp_exporter_service_name }}"
+snmp_exporter_service_template_owner: root
+snmp_exporter_service_template_group: root
+snmp_exporter_service_template_mode: 0444

roles/snmp_exporter/handlers/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: restart snmp_exporter
+  systemd:
+    name: "{{ snmp_exporter_service_name }}"
+    daemon_reload: true
+    state: restarted

roles/snmp_exporter/tasks/configure.yaml

@@ -0,0 +1,55 @@
+---
+- name: create group
+  ansible.builtin.group:
+    name: "{{ snmp_exporter_group_name }}"
+    system: true
+
+- name: create user
+  ansible.builtin.user:
+    name: "{{ snmp_exporter_user_name }}"
+    shell: "{{ snmp_exporter_user_shell }}"
+    home: "{{ snmp_exporter_user_home }}"
+    system: true
+    group: "{{ snmp_exporter_group_name }}"
+
+- name: create var path
+  ansible.builtin.file:
+    path: "{{ snmp_exporter_var_dir_path }}"
+    owner: "{{ snmp_exporter_var_dir_path_owner }}"
+    group: "{{ snmp_exporter_var_dir_path_group }}"
+    mode: "{{ snmp_exporter_var_dir_path_mode }}"
+    state: "{{ snmp_exporter_var_dir_path_state }}"
+
+- name: create etc path
+  ansible.builtin.file:
+    path: "{{ snmp_exporter_etc_dir_path }}"
+    owner: "{{ snmp_exporter_etc_dir_path_owner }}"
+    group: "{{ snmp_exporter_etc_dir_path_group }}"
+    mode: "{{ snmp_exporter_etc_dir_path_mode }}"
+    state: "{{ snmp_exporter_etc_dir_path_state }}"
+
+- name: configure
+  ansible.builtin.template:
+    src: "{{ snmp_exporter_config_file_template_src }}"
+    dest: "{{ snmp_exporter_config_file_template_dest }}"
+    owner: "{{ snmp_exporter_config_file_template_owner }}"
+    group: "{{ snmp_exporter_config_file_template_group }}"
+    mode: "{{ snmp_exporter_config_file_template_mode }}"
+  notify:
+    - restart snmp_exporter
+
+- name: configure systemd unit
+  ansible.builtin.template:
+    src: "{{ snmp_exporter_service_template_src }}"
+    dest: "{{ snmp_exporter_service_template_dest }}"
+    owner: "{{ snmp_exporter_service_template_owner }}"
+    group: "{{ snmp_exporter_service_template_group }}"
+    mode: "{{ snmp_exporter_service_template_mode }}"
+  notify:
+    - restart snmp_exporter
+
+- name: manage service
+  ansible.builtin.service:
+    name: "{{ snmp_exporter_service_name }}"
+    enabled: "{{ snmp_exporter_service_enabled | default(true) }}"
+    state: "{{ snmp_exporter_service_state | default('started') }}"

roles/snmp_exporter/tasks/install.yaml

@@ -0,0 +1,56 @@
+---
+- name: determine install status
+  ansible.builtin.stat:
+    path: "{{ snmp_exporter_opt_dir_path }}/snmp_exporter"
+  register: st
+
+- name: create opt path
+  ansible.builtin.file:
+    path: "{{ snmp_exporter_opt_dir_path }}"
+    owner: root
+    group: root
+    mode: 0755
+    state: directory
+    
+- block:
+  - name: download
+    ansible.builtin.get_url:
+      url: "{{ snmp_exporter_release_url }}"
+      dest: "{{ snmp_exporter_download_path }}"
+      checksum: "{{ snmp_exporter_checksums[snmp_exporter_release_file] }}"
+    register: dl
+    until: dl is success
+    retries: 5
+    delay: 10
+
+  - name: extract
+    ansible.builtin.unarchive:
+      src: "{{ snmp_exporter_download_path }}"
+      dest: "{{ snmp_exporter_unarchive_dest_path }}"
+      remote_src: true
+
+  - name: install
+    ansible.builtin.copy:
+      src: "{{ snmp_exporter_extracted_path }}/{{ item }}"
+      dest: "{{ snmp_exporter_opt_dir_path }}/{{ item }}"
+      remote_src: true
+    loop: "{{ snmp_exporter_binaries }}"
+  when: not st.stat.exists
+
+- name: permissions
+  ansible.builtin.file:
+    path: "{{ snmp_exporter_opt_dir_path }}/{{ item }}"
+    owner: root
+    group: root
+    mode: 0755
+  loop: "{{ snmp_exporter_binaries }}"
+
+- name: symlink
+  ansible.builtin.file:
+    src: "{{ snmp_exporter_opt_dir_path }}/{{ item }}"
+    dest: "/usr/local/bin/{{ item }}"
+    owner: root
+    group: root
+    mode: 0755
+    state: link
+  loop: "{{ snmp_exporter_binaries }}"

roles/snmp_exporter/tasks/main.yaml

@@ -0,0 +1,28 @@
+---
+- name: gather os specific variables
+  ansible.builtin.include_vars: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: include os specific tasks
+  ansible.builtin.include_tasks: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- ansible.builtin.include_tasks: install.yaml
+
+- ansible.builtin.include_tasks: configure.yaml

roles/snmp_exporter/templates/snmp_exporter.service.j2

@@ -0,0 +1,21 @@
+# {{ ansible_managed }}
+
+[Unit]
+Description=SNMP Exporter
+After=network-online.target
+
+[Service]
+User={{ snmp_exporter_user_name }}
+Restart=on-failure
+ExecStart={{ snmp_exporter_bin_path }} \
+{% for arg in snmp_exporter_bin_args %}
+  {{ arg }} {% if not loop.last %}\{{ "\n"}}{% endif %}
+{% if loop.last %}
+
+{% endif %}
+{% endfor %}
+
+WorkingDirectory={{ snmp_exporter_var_dir_path }}
+
+[Install]
+WantedBy=multi-user.target

roles/tailscale/defaults/main.yaml

@@ -0,0 +1,10 @@
+---
+# tailscale_package_name: tailscale
+# tailscale_package_state: present
+
+# tailscale_service_name: tailscaled
+# tailscale_service_state: started
+# tailscale_service_enabled: true
+
+tailscale_up_args:
+  []

roles/tailscale/tasks/Debian.yaml

@@ -0,0 +1,13 @@
+---
+- name: add tailscale repo
+  block:
+    - name: install apt key
+      ansible.builtin.get_url:
+        url: "https://pkgs.tailscale.com/stable/{{ ansible_distribution | lower }}/{{ ansible_distribution_release | lower }}.noarmor.gpg"
+        dest: /etc/apt/trusted.gpg.d/tailscale-archive-keyring.gpg
+
+    - name: install apt repo
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/etc/apt/trusted.gpg.d/tailscale-archive-keyring.gpg] https://pkgs.tailscale.com/stable/{{ ansible_distribution | lower }} {{ ansible_distribution_release | lower }} main"
+        state: present
+        filename: tailscale

roles/tailscale/tasks/configure.yaml

@@ -0,0 +1,11 @@
+---
+- name: manage service
+  ansible.builtin.service:
+    name: "{{ tailscale_service_name | default('tailscaled') }}"
+    state: "{{ tailscale_service_state | default('started') }}"
+    enabled: "{{ tailscale_service_enabled | default(true) }}"
+
+- name: tailscale up
+  ansible.builtin.shell:
+    cmd: "tailscale up {{ tailscale_up_args | join(' ') }} --authkey {{ tailscale_authkey }}"
+  no_log: true

roles/tailscale/tasks/install.yaml

@@ -0,0 +1,5 @@
+---
+- name: install
+  ansible.builtin.package:
+    name: "{{ tailscale_package_name | default('tailscale') }}"
+    state: "{{ tailscale_package_state | default('present') }}"

roles/tailscale/tasks/main.yaml

@@ -0,0 +1,31 @@
+---
+- name: gather OS specific variables
+  ansible.builtin.include_vars: "{{ lookup('ansible.builtin.first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: run os specific tasks
+  ansible.builtin.include_tasks: "{{ lookup('ansible.builtin.first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- debug:
+    var: ansible_facts
+
+- include_tasks: install.yaml
+
+- include_tasks: configure.yaml

roles/ufw/defaults/main.yaml

@@ -0,0 +1,6 @@
+---
+# ufw_state: enabled
+# ufw_policy: allow
+ufw_rules:
+  - port: ssh
+    rule: allow

roles/ufw/tasks/configure.yaml

@@ -0,0 +1,12 @@
+---
+- name: set ufw state
+  community.general.ufw:
+    state: "{{ ufw_state | default('enabled') }}"
+    policy: "{{ ufw_policy | default('allow') }}"
+
+- name: configure rules
+  community.general.ufw:
+    port: "{{ item.port | default(omit) }}"
+    proto: "{{ item.proto | default(omit) }}"
+    rule: "{{ item.rule | default(omit) }}"
+  loop: "{{ ufw_rules | default([]) }}"

roles/ufw/tasks/main.yaml

@@ -0,0 +1,26 @@
+---
+- name: gather OS specific variables
+  ansible.builtin.include_vars: "{{ lookup('ansible.builtin.first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: run os specific tasks
+  ansible.builtin.include_tasks: "{{ lookup('ansible.builtin.first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- include_tasks: configure.yaml

roles/unattended-upgrades/defaults/main.yaml

@@ -21,10 +21,10 @@ unattended_upgrades_unattended_upgrade: true
 unattended_upgrades_allowed_origins:
   - "${distro_id}:${distro_codename}"
   - "${distro_id}:${distro_codename}-security"
-  - "${distro_id}ESM:${distro_codename}"
+  - "${distro_id}:${distro_codename}-updates"
-  # - "${distro_id}:${distro_codename}-updates"
-  # - "${distro_id}:${distro_codename}-proposed"
   # - "${distro_id}:${distro_codename}-backports"
+  # - "${distro_id}:${distro_codename}-proposed"
+  # - "${distro_id}ESM:${distro_codename}"
 
 # List of packages to not update (regexp are supported)
 # unattended_upgrades_package_blacklist: []

roles/util/defaults/main.yaml

@@ -42,7 +42,7 @@ util_packages:
     - p7zip
     - p7zip-full
     - pigz
-    - pxz
+    - pixz
     - zstd
     - pbzip2
     - pv

roles/wireguard/templates/wg-multi.conf.j2

@@ -15,6 +15,9 @@ Address = {{ address }}
 {%   if "listen_port" in i %}
 ListenPort = {{ i.listen_port }}
 {%   endif %}
+{%   if "table" in i %}
+Table = {{ i.table }}
+{%   endif %}
 {% endmacro -%}
 
 {%- macro render_peer(p) %}
@@ -35,6 +38,12 @@ AllowedIPs = {{ p.allowed_ips }}
 AllowedIPs = {{ p.allowed_ips | join(', ') }}
 {%     endif %}
 {%   endif %}
+{%   if "preshared_key" in p %}
+PresharedKey = {{ p.preshared_key }}
+{%   endif %}
+{%   if "persistent_keepalive" in p %}
+PersistentKeepalive = {{ p.persistent_keepalive }}
+{%   endif %}
 {% endmacro -%}
 
 {% if wireguard_interfaces[_wireguard_interface] and