Add role for ufw

Add draft roles for cloudflared and tailscale
Add vault for monitor_servers
2024-04-14 18:32:50 -05:00 · 2024-04-14 18:31:59 -05:00 · 2024-04-14 18:31:11 -05:00 · 2024-04-14 18:30:40 -05:00 · 2024-04-14 18:30:16 -05:00 · 2024-04-14 18:30:05 -05:00
169 changed files with 33947 additions and 1275 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -0,0 +1,12 @@
+---
+kind: pipeline
+name: default
+
+steps:
+  - name: lint
+    image: python
+    commands:
+      - pip install yamllint
+      - pip install ansible-lint
+      - yamllint .
+      - ansible-lint .
--- a/files/grafana/dashboards/system.json
+++ b/files/grafana/dashboards/system.json
--- a/files/nsd/zones/cavi.cc.zone
+++ b/files/nsd/zones/cavi.cc.zone
@ -1,18 +0,0 @@
-; cavi.cc [320470]
-$TTL 86400
-@	IN	SOA	ns1.linode.com. hostmaster.kill0.net. 2022020501 14400 14400 1209600 86400
-@		NS	ns1.linode.com.
-@		NS	ns2.linode.com.
-@		NS	ns3.linode.com.
-@		NS	ns4.linode.com.
-@		NS	ns5.linode.com.
-@			MX	10	in1-smtp.messagingengine.com.
-@			MX	20	in2-smtp.messagingengine.com.
-@			TXT	"v=spf1 include:spf.messagingengine.com -all"
-default._domainkey			TXT	"v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDY8s2MeBFqZIwItrdDo4J0N0AIoNtf7Ui6jtyIqqs2if2D1h3Ee37McBxZhJ79TX3TZyXci/G0+DZm/F9w2Ye703JNmgjSo6V1fx3MMZicohnTwYs3yQScdWNjJ8ML6SEJtveIjIws2CQ4/Y8J3f6ilWh2OAUrRIAg2u/BV5odgwIDAQAB"
-mesmtp._domainkey			TXT	"v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDUDHvhiTIEgdxTfvcrL1znWbMBWC10L8enkWJmatLs1vGkEQkNbaK55aO3wDwuVZq9f2KmcEUA/GRUOJQy3XGu1xgPjVmR6Hqbx4ygjoAcMm8UfNc7UA8deKV8qCGEF2ag82n9LpDYcEQSehC/kE4bbUFaZk3FMUdTwMu5vB0vVQIDAQAB"
-_dmarc			TXT	"v=DMARC1; p=reject; adkim=s; aspf=s"
-@   A   45.33.21.121
-@   AAAA    2600:3c00::f03c:92ff:feb0:e05c
-www   A   45.33.21.121
-www   AAAA    2600:3c00::f03c:92ff:feb0:e05c
--- a/files/nsd/zones/chill9.com.zone
+++ b/files/nsd/zones/chill9.com.zone
@ -1,12 +0,0 @@
-; chill9.com [726945]
-$TTL 86400
-@ IN  SOA ns1.linode.com. hostmaster.kill0.net. 2022051201 14400 14400 1209600 86400
-@   NS  ns1.linode.com.
-@   NS  ns2.linode.com.
-@   NS  ns3.linode.com.
-@   NS  ns4.linode.com.
-@   NS  ns5.linode.com.
-@   A   45.33.21.121
-@   AAAA    2600:3c00::f03c:92ff:feb0:e05c
-www   A   45.33.21.121
-www   AAAA    2600:3c00::f03c:92ff:feb0:e05c
--- a/files/nsd/zones/chill9.net.zone
+++ b/files/nsd/zones/chill9.net.zone
@ -1,12 +0,0 @@
-; chill9.net [726945]
-$TTL 86400
-@ IN  SOA ns1.linode.com. hostmaster.kill0.net. 2022051201 14400 14400 1209600 86400
-@   NS  ns1.linode.com.
-@   NS  ns2.linode.com.
-@   NS  ns3.linode.com.
-@   NS  ns4.linode.com.
-@   NS  ns5.linode.com.
-@   A   45.33.21.121
-@   AAAA    2600:3c00::f03c:92ff:feb0:e05c
-www   A   45.33.21.121
-www   AAAA    2600:3c00::f03c:92ff:feb0:e05c
--- a/files/nsd/zones/confabulator.net.zone
+++ b/files/nsd/zones/confabulator.net.zone
@ -1,12 +0,0 @@
-; confabulator.net [307550]
-$TTL 86400
-@	IN	SOA	ns1.linode.com. hostmaster.kill0.net. 2022051201 14400 14400 1209600 86400
-@		NS	ns1.linode.com.
-@		NS	ns2.linode.com.
-@		NS	ns3.linode.com.
-@		NS	ns4.linode.com.
-@		NS	ns5.linode.com.
-@   A   45.33.21.121
-@   AAAA    2600:3c00::f03c:92ff:feb0:e05c
-www   A   45.33.21.121
-www   AAAA    2600:3c00::f03c:92ff:feb0:e05c
--- a/files/nsd/zones/ctrl-v.org.zone
+++ b/files/nsd/zones/ctrl-v.org.zone
@ -1,16 +0,0 @@
-; ctrl-v.org [687762]
-$TTL 86400
-@ IN  SOA ns1.linode.com. hostmaster.kill0.net. 2022051201 14400 14400 1209600 86400
-@   NS  ns1.linode.com.
-@   NS  ns2.linode.com.
-@   NS  ns3.linode.com.
-@   NS  ns4.linode.com.
-@   NS  ns5.linode.com.
-@     MX  10  in1-smtp.messagingengine.com.
-@     MX  20  in2-smtp.messagingengine.com.
-@     TXT "v=spf1 include:spf.messagingengine.com include:mailgun.org -all"
-mesmtp._domainkey     TXT "v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ihB/cUM+FkgYv5MPoZQQQLLFfu77bPYgQv64g1xjNw0c3jmHMKjQ51zW5lbvu/DAwKxtZqHjnruyvcLzRGcWzeV8udk88l+DuskTbIAYn0U5tU0fzTRwiARz4flik+JQtA0P+jvK5jCjmmEHpz6QUa+UN6rZKpz1jB3SgXXbpwIDAQAB"
-@   A   45.33.21.121
-@   AAAA    2600:3c00::f03c:92ff:feb0:e05c
-www   A   45.33.21.121
-www   AAAA    2600:3c00::f03c:92ff:feb0:e05c
--- a/files/nsd/zones/kill0.com.zone
+++ b/files/nsd/zones/kill0.com.zone
@ -1,12 +0,0 @@
-; kill0.com [726945]
-$TTL 86400
-@ IN  SOA ns1.linode.com. hostmaster.kill0.net. 2022051201 14400 14400 1209600 86400
-@   NS  ns1.linode.com.
-@   NS  ns2.linode.com.
-@   NS  ns3.linode.com.
-@   NS  ns4.linode.com.
-@   NS  ns5.linode.com.
-@   A   45.33.21.121
-@   AAAA    2600:3c00::f03c:92ff:feb0:e05c
-www   A   45.33.21.121
-www   AAAA    2600:3c00::f03c:92ff:feb0:e05c
--- a/files/nsd/zones/kill0.net.zone
+++ b/files/nsd/zones/kill0.net.zone
@ -1,27 +0,0 @@
-; kill0.net [726944]
-$TTL 86400
-@                       SOA ns1.linode.com. hostmaster.kill0.net. 2022053101 14400 14400 1209600 86400
-@                       NS  ns1.linode.com.
-@                       NS  ns2.linode.com.
-@                       NS  ns3.linode.com.
-@                       NS  ns4.linode.com.
-@                       NS  ns5.linode.com.
-@                       MX  10  in1-smtp.messagingengine.com.
-@                       MX  20  in2-smtp.messagingengine.com.
-@                       TXT "v=spf1 include:mailgun.org ~all"
-mailo._domainkey        TXT "k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC7bl1IcQTV0h6yK7wAsuLqj6LjlTxL1ktnGMdeC+J0hlDOHQPey7XEjL9Hj1Ye55Fu1cyBNx7eYn/mLZgiuGu9MccbgIyzRasP1DHG2mQ9omi8z7igesKtRaasyJ4EM6oM3BNSmDneVcInxOUa+6E9fJCesT+X3Flf8XEvuV3gowIDAQAB"
-jump0                   A       45.33.21.121
-jump1                   A       198.58.98.26
-mine0                   A       173.255.193.88
-vpn-home            300 A       98.52.91.99
-vpn-jump0               A       45.33.21.121
-jump0                   AAAA    2600:3c00::f03c:92ff:feb0:e05c
-jump1                   AAAA    2600:3c00::f03c:93ff:feac:0daf
-mine0                   AAAA    2600:3c00::f03c:92ff:fe70:d8d1
-git                     CNAME   jump0.kill0.net.
-monitor                 CNAME   jump0.kill0.net.
-ping                    CNAME   jump0.kill0.net.
-stats                   CNAME   jump0.kill0.net.
-dl                      CNAME   jump0.kill0.net.
-ping-home           300 A       98.52.91.99
-ping-home           300 AAAA    2001:558:6033:96:4ea:10a5:9c40:3d9f
--- a/group_vars/all/main.yaml
+++ b/group_vars/all/main.yaml
@ -102,17 +102,17 @@ rsyslog_archival_format_enabled: true
 rsyslog_outputs:
  - name: omfwd
    params:
-      #target: 127.254.254.1
-      target: 10.255.0.1
-      #port: 1514
+      target: 169.254.0.1
      port: 514
      protocol: tcp
      action.resumeretrycount: -1
      queue.type: linkedlist
-      queue.size: 10000
+      queue.size: 1000000
      queue.filename: fwd
      queue.saveonshutdown: "on"
      keepalive: "on"
+      template: RSYSLOG_SyslogProtocol23Format
+      tcp_framing: octet-counted

 sudo_aliases:
  host:
@ -210,17 +210,17 @@ teleport_config:
 firewall_ipset_node_exporter:
  - "{{ lookup('dig', 'jump0.kill0.net./A') }}"
  - "{{ lookup('dig', 'jump0.kill0.net./AAAA') }}"
-  - 10.255.0.1
+  - 169.254.0.1

 firewall_ipset_blackbox_exporter:
  - "{{ lookup('dig', 'jump0.kill0.net./A') }}"
  - "{{ lookup('dig', 'jump0.kill0.net./AAAA') }}"
-  - 10.255.0.1
+  - 169.254.0.1

 firewall_ipset_mtail:
  - "{{ lookup('dig', 'jump0.kill0.net./A') }}"
  - "{{ lookup('dig', 'jump0.kill0.net./AAAA') }}"
-  - 10.255.0.1
+  - 169.254.0.1

 node_exporter_du_directories:
  - /var/log/syslog
@ -230,7 +230,7 @@ wireguard_iptables:
  wg0:
    input: true

-wireguard_network_prefix: 10.255.0
+wireguard_network_prefix: 169.254.0
 wireguard_peers:
  wg0:
  - public_key: 1ipGUnK8XDbIoBIEF440BhwLUe0yHa5l3kZZc4eFxV8=
@ -241,57 +241,125 @@ supervisor_unix_http_server_socket_chown: root:node_exporter
 supervisor_unix_http_server_socket_chmod: "0770"

 firewall_ipset_loki:
-  - 10.255.0.1
+  - 169.254.0.0/24
+
+firewall_ipset_promtail:
+  - "{{ lookup('dig', 'jump0.kill0.net./A') }}"
+  - "{{ lookup('dig', 'jump0.kill0.net./AAAA') }}"
+  - 169.264.0.0/24

 promtail_clients:
-  - url: http://10.255.0.1:3100/loki/api/v1/push
+  - url: http://169.254.0.1:3100/loki/api/v1/push
+    external_labels:
+      region: dallas
+      provider: linode

 promtail_scrape_configs:
- job_name: system
-  static_configs:
-  - targets:
-      - localhost
+- job_name: journal
+  journal:
+    json: false
+    max_age: 12h
+    path: /var/log/journal
    labels:
-      job: syslog
-      __path__: "/var/log/syslog/{{ ansible_hostname }}/**/*.log"
- job_name: nginx
-  static_configs:
-  - targets:
-      - localhost
-    labels:
-      job: nginx
-      host: "{{ ansible_hostname }}"
-      __path__: /var/log/nginx/*.log
+      job: systemd-journal
+  relabel_configs:
+    - source_labels:
+      - __journal__systemd_unit
+      target_label: systemd_unit
+    - source_labels:
+      - __journal_unit
+      target_label: unit
+    - source_labels:
+      - __journal_priority_keyword
+      target_label: priority
+    - source_labels:
+      - __journal_syslog_identifier
+      target_label: syslog_identifier
  pipeline_stages:
    - match:
-      selector: '{job="nginx"}'
+        selector: '{systemd_unit=~"(alertmanager|blackbox_exporter|grafana|karma|kthxbye|loki|mimir|node_exporter|prometheus|promtail|pushgateway|thanos).+"}'
+        stages:
+          - logfmt:
+              mapping:
+                level:
+                ts:
+          - timestamp:
+              source: ts
+              format: RFC3339Nano
+          - timestamp:
+              source: t
+              format: RFC3339Nano
+          - labels:
+              priority: level
+- job_name: nginx-access
+  static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: nginx-access
+        __path__: /var/log/nginx/*.access.log
+  pipeline_stages:
+    - match:
+        selector: '{job="nginx-access"}'
        stages:
          - regex:
-          expression: '^(?P<remote_addr>[^ ]+) - (?P<remote_user>[^ ]*) \[(?P<time_local>.*)\] "(?P<method>[^ ]*) (?P<request>[^ ]*) (?P<protocol>[^ ]*)" (?P<status>[\d]+) (?P<body_bytes_sent>[\d]+) "(?P<http_referer>[^"]*)" "(?P<http_user_agent>[^"]*)"?'
-      - metrics:
-          nginx_requests_total:
-            type: Counter
-            description: requests in nginx access logs
-            source: method
-            config:
-              action: inc
+              expression: ^(?P<hostname>[0-9A-Za-z\.:-]+) (?P<remote_addr>[0-9A-Za-z\.:-]+) (?P<remote_logname>[0-9A-Za-z-]+) (?P<remote_username>[0-9A-Za-z-]+) \[(?P<timestamp>\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} (\+|-)\d{4})\] "(?P<request_method>[A-Z]+) (?P<URI>\S+) (?P<http_version>HTTP\/[0-9\.]+)" (?P<request_status>\d{3})
+          - timestamp:
+              source: timestamp
+              format: "02/Jan/2006:15:04:05 -0700"
          - labels:
-          #remote_addr:
-          #remote_user:
-          #time_local:
-          method:
-          #request:
-          #protocol:
-          status:
-          body_bytes_sent:
-          #http_referer:
-          #http_user_agent:
+              hostname:
+              method: request_method
+              status: request_status
+              version: http_version
+- job_name: nginx-error
+  static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: nginx-error
+        __path__: /var/log/nginx/*.error.log
+  pipeline_stages:
+    - match:
+        selector: '{job="nginx-error"}'
+        stages:
+          - regex:
+              expression: '^(?P<timestamp>\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}) \[(?P<priority>\w+)\] (?P<pid>\d+)\#(?P<tid>\d+): (?:\*(?P<cid>\d+))?'
+          - labels:
+              priority:
+          - timestamp:
+              source: timestamp
+              format: "2023/08/16 02:43:32"
+          - regex:
+              expression: 'host: "(?P<hostname>[0-9A-Za-z\.:-]+)"'
+          - labels:
+              hostname:
+- job_name: syslog
+  syslog:
+    listen_address: 0.0.0.0:1514
+    listen_protocol: tcp
+    idle_timeout: 60s
+    label_structured_data: true
+    labels:
+      job: syslog
+  pipeline_stages:
+    - match:
+        selector: '{host=~"ap0|coresw0|fw0|power0|172\\."}'
+        stages:
+          - static_labels:
+              region: home
+              provider: home

-loki_service_enabled: false
-loki_service_state: stopped
-
-promtail_service_enabled: false
-promtail_service_state: stopped
+  relabel_configs:
+   - source_labels:
+       - __syslog_message_hostname
+     target_label: host
+   - source_labels:
+       - __syslog_message_severity
+     target_label: priority
+   - source_labels:
+       - __syslog_message_app_name
+     target_label: syslog_identifier

 influxdb_service_enabled: false
 influxdb_service_state: stopped
@ -300,3 +368,7 @@ influxdb_package_state: absent
 telegraf_service_enabled: false
 telegraf_service_state: stopped
 telegraf_package_state: absent
+
+lego_credential_files:
+  - name: credentials.json
+    content: "{{ vault_lego_gcp_service_account | string }}"
--- a/group_vars/all/vault.yaml
+++ b/group_vars/all/vault.yaml
--- a/group_vars/jump_servers/main.yaml
+++ b/group_vars/jump_servers/main.yaml
@ -7,7 +7,7 @@ firewall_allowed_udp_ports:
  - 1194

 firewall_ipset_syslog:
-  - 10.255.0.0/24
+  - 169.254.0.0/24

 autossh_authorized_keys:
  - key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOvKqDI6VUYFgMUC54pVr5U8CX+Xl2ewV7PIYkTiQ70o
@ -46,7 +46,7 @@ telegraf_config_d:
          name_override: ping6
          binary: ping6
        - urls:
-            - 10.255.0.1
+            - 169.254.0.1
          count: 10
          ipv6: false
          binary: ping4
@ -311,43 +311,49 @@ wireguard_peers:
  wg0:
    - comment: mine0.kill0.net
      public_key: Cm9yZNczjghAh4hV4fSvy3rsmuLsQFZk+ET5CoWxVnI=
-      #endpoint: "{{ lookup('dig', 'mine0.kill0.net./A') }}:{{ wireguard_port }}"
      endpoint: "mine0.kill0.net:{{ wireguard_port }}"
-      allowed_ips: "{{ hostvars['mine0.kill0.net'].wireguard_interfaces.wg0.address }}"
-    - comment: vpn-home.kill0.net
-      public_key: j5AgKWcXx8we7QVkt6//oQWsGfXj+5IJKt9mx0EpTW0=
-      endpoint: "vpn-home.kill0.net:{{ wireguard_port }}"
-      allowed_ips: 172.16.0.0/16, 10.255.0.2/32
-  wg1:
-    - comment: pixel-2
-      public_key: GzQOU0x1POvkY4+6smBGkE/B1XytoVxIJa6zGX8j6Bc=
      allowed_ips:
-        - 192.168.255.16/32
-        - 2600:3c00:e000:343::10/128
+        - 169.254.0.2/32
+        - fc00::ffff:169.254.0.2/128
+    - comment: vpn-home.kill0.net
+      allowed_ips:
+        - 172.16.0.0/16
+        - 169.254.0.16/32
+        - fc00::ffff:169.254.0.16/128
+      endpoint: "vpn-home.kill0.net:{{ wireguard_port }}"
+      persistent_keepalive: 25
+      preshared_key: "{{ vault_wireguard_preshared_key.home }}"
+      public_key: fUSQ7Uxkxij/0p+SIRekb6moqW0t/qdFaP2HsjRsNRs=
+    - comment: retropie
+      allowed_ips:
+        - 172.31.0.0/16
+        - 169.254.0.17/32
+        - fc00::ffff:169.254.0.17/128
+      persistent_keepalive: 25
+      preshared_key: "{{ vault_wireguard_preshared_key.retropie }}"
+      public_key: lLvracXkf8HNfgKpJkzei9ys58aAs4DT3Z3bjNRFsQY=
+  wg1:
+    - comment: pixel
+      public_key: zCDfH5Eqv0oRNWC8TtrkGby3+BAtiQtXxbsmA/lZtXQ=
+      allowed_ips:
+        - 192.168.255.16/24
+        - fc01::ffff:192.168.255.16/128
+        - 2600:3c00:e000:343::ffff:192.168.255.16/128
    - comment: work laptop
      public_key: TRT1SRQd3mFJDJK9tdglqsydXJmkzyrNdUOm4nr7M3k=
      allowed_ips:
-        - 192.168.255.17/32
-        - 2600:3c00:e000:343::11/128
-    - comment: home workstation
-      public_key: ISvgu8zZWjmKyKrJi2mbqoJg2mrvIjPbQRs0Sp+dLzc=
-      allowed_ips:
-        - 192.168.255.18/32
-        - 2600:3c00:e000:343::12/128
-    - comment: rick
-      public_key: oFJcRhs7tQ4vPHTjbKwwWirpjx9T9ng7PFj3+iAVYWo=
-      allowed_ips:
-        - 192.168.255.32/32
-        - 2600:3c00:e000:343::20/128
+        - 192.168.255.17/24
+        - fc01::ffff:192.168.255.17/128
+        - 2600:3c00:e000:343::ffff:192.168.255.17/128

 unbound_interfaces:
  - 127.0.0.1
  - 192.168.255.1
  - ::1
-  - 2600:3c00:e000:343::1
+  - 2600:3c00:e000:343::ffff:192.168.255.1

 unbound_access_control:
  - 127.0.0.1 allow
  - 192.168.255.0/24 allow
  - ::1 allow
-  - 2600:3c00:e000:343::/64 allow
+  - 2600:3c00:e000:343::ffff:192.168.255.0/120 allow
--- a/group_vars/jump_servers/vault.yaml
+++ b/group_vars/jump_servers/vault.yaml
@ -1,223 +1,230 @@
 $ANSIBLE_VAULT;1.1;AES256
-36396137393836323465386631643461656431316666376562623633383965393863383866663764
-3664343734343065343236303365373962333162306564620a623362326163393766343735653061
-64393932383066323264636530613036353637343231666439346234663430326366396532663765
-3536663666643838360a316462376363613562373965653536333763386635343362393938386331
-39663266616365383166393232646530656135373234646166393365343233666635393430313136
-66616361636638323430343334643230623331623334343162333335353265333436326239626664
-30623039333737383531663738616337396136353836383537343337316565623562393235303566
-63656234663765313062666435313431633861646137313330386633383062656335336639633631
-31386561376365623634666231643134663230643736376662356361313464666638363961366437
-61323033386661356561653961623333353637613439666437333164643532343863333434613061
-63646432396333303965663730623061333065653432326136333337633862393339363130373138
-36366163316635383336316537393761633962336138643139386638373134313635336666303765
-62316531336165323965343232636339313462633536623139303865663862376364363261363865
-31353064646338646662386639343462386639393162363334363937363337613963313135663365
-66343365363232623564613035303139663937356430336537346564643134313763393462323638
-30616462363661623466663162333834323937623335316261646533316137613564316532653165
-33343133376538643961656364656666346533316336626464663939313137643461303232666162
-32353131353864373738396335613763366639633837653636386139393862616364613265313935
-62353134303733393836666337393530643465343333373230346133396163623332336131323730
-39383264303935343763343033303864316433613334633137333031626563393233663932376434
-66303638643232376633636331613234316339666630393534333136306639616662613361663031
-31316630323338383061346333633063393261353463623039633063633132623730303161663531
-65353030303763336639636265663333333639306432306662386232303439626235663433376437
-37336461376662663035373336663937333132383964396561626337626632303064656365313633
-61663630316163323163383436636636313333353437646330346532656236626562663332323636
-65303430663133363464323262313531376531303739613364336262393965376533343136323034
-65376461326362313732323730353137663036393835333939353962643338326162306163626536
-37316262623265633363356435316632653466636137303131303664636433376236613237376339
-36616639643232356330393134333364303137633736633764346233636330386232316566366435
-30613261613936343738303763623966653936323661383164613933333633653339363535306138
-32326466306634633965666466393435656432336163663130666266363230653730396665623531
-36643364306537306663303537333063363565386337663061623661343838303638393965373165
-38613939613061376161626163336164656237356164303562376137633135613738386331323262
-30373539633630646339323930373737346136633465616535643439643134306430653062383664
-61313138376138373961376561303162616438663263653561363339396132393834373566663436
-62356331323465616134656237356434633830666231646434363664623139373737393830616338
-36353066613464353739336462623966356330653534366332663735663937306462393233383939
-36363066633563393463303363653631646464323937613234333835306139373462366661643961
-30316462636638353531336266633061663933316266303335623837376239633835663265336338
-39313334396565653262613736616536646461656438373839316337363963663135353261353133
-32373366366236353663393065306338373961636432353533386436666532313637306433373236
-38383037663037643763383465313862336334326637346338383235663061316232613365656266
-31616136373135323039313633373538353761663439323839313365313462663063373339623530
-61313731303861333631613464343232303763316462643935626366346130366531313631626630
-39636630663866336161623835666261366337376239653139613230616231353636616266663238
-31653466363530346262326630353661366635616162313733323032633736653362306665363565
-31653731343465373736646338383830393735643736646266323965356336393939366537386566
-35613561333834653834626233396133323337303439643432373931616237613439343665343061
-39666661353532326435373332393739356636636433623163383337663165613834393864303533
-32356336366336353261653235663666633335626331663964636263656136366232373838613962
-37393464376137663630333334363234393464313062353366656435646633653265616265383535
-61333061303633623065666366643037333139356465343932376664333163623532626331336139
-33373732613264636331623964393336383665613264343131613138386362386362343539346234
-30336237356436623262393139363538306530356530353237666339386565613931303131666262
-30363866393061663437633532356238383530363066623862393531366530613731393137343434
-33386434613632383066636638356161323837653630363830336233653830343261303933616565
-65313334633838663264623032656131646331613539666436343334663061313837353030626161
-63303362666662356235343065373231646334656565316564626234363431346664373036303333
-39343363346365323237356365323062313630323736323737643432353262366534653131313033
-63383638333334333361383461626361333766343861653538343562326366623332626131613136
-62643537636233383263656564306430386333346432353434623433373638366536393438333434
-37656539303736633938316462366230613131633936363034386639623330653535326264333861
-35616537623461316662636166613530373963316236393938363932616566333430613366626363
-66383139323565353830303466356233353066316663653732303534383765346636653132363130
-32303563353232616537613966663836623832383335646331616364353336313363313234323362
-66616136636533346339363563623734623239626230636565623338363861393338613337623530
-64626363343533303333626234326666623136333332323532383662663635633538313433303835
-34623134386631376639623639313164393033616664346338633033656630623436633130373665
-38356635396238613633333738326233663933666562356630613063303230353462653264393531
-31303736633030663761376134366631646130363139623465653661366335363830633566333237
-33376631343334376435386135653330343832353339313931323434303265343361336231643638
-66623539313162643337353432393865626538633265633363353830306663393233333962313636
-33333565356536376464653131376633353363316663336563323230326537613165353134366365
-61363030326334656139353938613531643864316434383266353633373735326562306239323961
-37336638663837333738313230316236346262326135346536343331356234313130353661383464
-35376236346366373363326138383430323132626663303138353938383263643665393839363162
-31366166613037383166313264373035663066336138623535313035303533613132613436313136
-66393764333732356333363462333366346363613262316130636235353361313731383839653563
-63383134643262636262666237356233393430336163613135623264633336396139646231363562
-34393031663961643562396234666437356665356331633834396637336264653265353065306233
-30393461313663313564373236663362353435393535306465353136613730333866636639633161
-30666566393266616134636264366666356438616632336661393639366635356262653832353633
-32623466303835633065613936373063626432326463336163303838613836646332643035653933
-63363630663161373039653330633631643638313036633537323364373739363736656231636535
-35396466373666353361366535366334313538313639663131336662386166316162326331373838
-34386232653930383133613164393435346661643466343762343463376537633036393366656164
-34366465613839623533363235343737333565326165633634386230323938646166643737333261
-64333139663463666432346461613033616539643463323263343563303361373539303834353434
-61306635323463383238633738303830646263663036396566336534623237636234303566643533
-39663462663063386137326630353164633561653936343665326665306665326238303230346436
-31633138303236666362306162663036386334623339656565353730643630396263363738306139
-64323230616164303638643263396432646438356534313433633536656432333738303038323266
-31643965383036326134653030333932323231313363336263656534303839346631636230323032
-61303033383932626238353466353631326633633565343065306561396636393835373966383032
-61363061653662373731313862326461373133343930393963343062623663633033323865323565
-62633736623365613631326464373662393861663737623836666532353339363232363630333662
-65333265386561336337353838353238316466336162393738623034376339653864393733643837
-38313763656431323261366634386331366262653838613036646633326464383565353136356566
-32313131313466613266643435663933646132646339353239343535363333393535346565383331
-32326566383337323662663438316639366139386433316639633463333661396337393837646435
-66313637653939626536326332306139393438333137323532316130636439313066383633396335
-38373062353930623661306339653234336135396233383965303861363535616633366666656562
-37336331316534656465613536313364346633393066323839393833393864363234356330663264
-65336263613861383837373533646430666539316638323966623761373633666437306432386235
-66353531303533323662613565363065356236383939623237363835616262326536373962343538
-30316631656465313264393932626232346637356531336536613561383434663934643432613164
-33313833613532613365393637323262346437343933353138623765626665656663306263393862
-39303865316537643063363665626465356631653534393462353830653931636563653333323733
-31343864333630366566613731366333323631313337636236653662613832626464626333363537
-33303762363332306266323538323366383863383033616563376231303937316163396638663162
-64386664313863636535366331646238626437353664313731346633353738343733626263666230
-30616161333061393061366430656330613737333133656637656664316265616365313436373939
-65653564326165303761326236343436326363383538613734303539363363316135653630666138
-38663333323863363163353838653765353937313166316230323961376136326438653866346665
-34306561356536663363666162643362316139313438323632366136366461663230613563613434
-37333838663239356236343731313430363232623633626364336664613839393036393566656366
-61616332666262336231363262333832613937313330373231383830343130323966333261353661
-34633661363731613430393262373839333863393730613730323866623837363936333039383535
-36353763313565633037393032386135376537343430363535376238376131653935366434346431
-33353338323935613638306234353963653438323031643735613035613335393834343961373037
-37653131333336353230636136633431333463316137333636363338333230656131346633326162
-33303635613033333730663162623965343230303533393065306539666439656361306634646662
-38616234326637393364303731303566363661633462393836633237353139616634373933356462
-66303864333133643238313061386538313430636231653265336463633437396134626238386365
-38646135363764373837376534386132616139396238373765316633336135396462646230396233
-38393432373736343236646364313037633032666631313462356164656465333837383037353038
-39343962646236363633323465636638656266323966393635373163323330613937656266326636
-64633666323061623266643939366630396237643731343531623031663363663963376336316334
-31323836366665386336313139613836353764343066633231306433363538393438366162376537
-38306436346662336262623832323964663138383262393262396366656465343731373135663562
-63316230366236376238346639613034656662623166306536303031313930343938363363626333
-35353837326134646535626164663762306431306464323230663763616465636435643064393830
-65663439343166376163346137666431653731313738623630623263643133353439363730623230
-34303265383164623530366334343066316361313533323831343833623634326661366532313265
-64333034636663383437666238346434313761366262626231666163373433343230623662653762
-37363234623932636536356565313062633131313334623364333262336561616334643534316666
-38623032376432616339343939646638303630326235316163363530326238306335656630336462
-36313234643064333737613661393164306263353438666334646164346430333665396665386436
-32643136323431303063306135363131373966343666616163326466656233386532383930343764
-34313536643663623031326236663866396165656539313461313933343035306336643631363261
-65333934333231373435376134643237343237636230386465663832363665333334316663303761
-32616133386637303437376639316261643938383563636433633035353138343137623838313466
-65643835643562303234373137323037643165393738366262633638323939653233666163646630
-31613863393832336663326266306430663864323031383161663762636535636238363663343066
-38306533663931623537363964323733666563663765656331306236353436646566343766313039
-37646334643839326531326132633433653030376437373734643038653732346335653161323932
-36616533346437373665636166313337353136616466383237396266373131353136313535323666
-63373034613961643531643936633566383231336166323762316539373334323134636332383232
-36383336656538386631393665336661393432373339323432636565613963656232623034656635
-63376161306631326632636232653831643636396365303762323661366166353539343939313561
-39616233643564656538303764366365326338303436303261656433313766373766383638333634
-66346464623565366530663163666339333636363463336564393034373564633565623535646136
-37613133346565363230653666356631343037636638343832663866613461333061313464373736
-37323563663634373931396232626436626533323566323463346535353362333262633764366664
-30373337666366313866656362613562656239653565613035323936383861663931616266313637
-31636631326630393834346237613965396534323366313039643566343133363537393632663264
-66366265623962353164336463373031323262323936383163613834643433616333306661613430
-62366464353464326636656234336433656633376636366139343338373161303965333637626661
-30336337343936356131303237393264363232653033363163363036376163336639353961343563
-35346336666335636266373861626465633733613032393438616434313735316132313665663635
-34326438316632346666636265633035383336336462656331353737623066313765373366396636
-37383366303764386566316261316232663163616234663966396665313138303839646262306338
-63363365333735626165373735333631363761663735356635386139393739313764623531326561
-61663936363437376261613266633163326366333730323063633436643037663631303537656363
-66633334623064643239336439613735333431363631333435373532316230623065316332336438
-37346336366466366335653562646265613033656466306632646566626666323337353336366366
-62346163383439363933633763376639386132313333616261346234343439653533333462663436
-65353165313865313635383538633432613565343136383665303064636434313135383236636436
-30626538303437623837343663396464666232393139656335613739356165616136316263323337
-38386537326132386264363066333730653863353430643633656533663262613963633231383533
-65623032356131313936623931333234303532626533316636633763393631313139326562616530
-37343965373835393564613630373632666437393738666633636536366135316336333565336538
-61636635633861353561353063666433343837313733653837653239393061313732373930323339
-33653965346230616336323766363434643030633166313562366561363963396663626239343834
-34663933373832666635643961613461643331346564323431343365343439626135613638343866
-65333732653366343032373833623566613865323539666463623163623937343338386632646330
-34393865333864343666376265353062383966653839316263376434636531366561316433373835
-63343264383465336439356565313130373736376532376538336533323134666565346261353435
-62343534313866343331346439303164633539336537613130353364353430323361383938323137
-38353862663730343234333566643936356562383632313238303166646438646435623765373362
-66323339656466653235346661353266383339616364613562656233653935653739323262353661
-35356338363035373066323238323364336438643839313435313163383935316163396335303231
-36303133636539316661396664376639653265376266366432326633323734313165356537656337
-61633835303735366332336134613733336534646531393265633437373862316262663066393262
-61646663363239633430363165346534386639383562316161363532396266613837346230323663
-33623539633637666362346332323833316165643436353332363038343436666536336461636130
-37383839393866386139343565373164626639326530666662323230373030333938393531326435
-61306436623362373363623135336139343162393236326463666664323465646436366561323331
-30396663643765396234346265353831623634343963393234306532613336353732373630363830
-31613561353464306363316136383463396361353933313239643732353335656232636230323539
-64316163316461666564353637626532363966313332353362383936643661363066353734666631
-62363562613362333436313534326135393665663930376535646562646635326236363163626632
-31376334336265323737326138373532323363393937303635373663653862393730646532616637
-34643235636165343063633836623936666564313566303861356332636130393635353438613637
-64303430653061356533373235336661363139643537633337386164303236613934313566643431
-65393664333233326565653634656566393738366566613137383436366638656561376135626364
-38303633343737633464356134616331366266613164386439346338373036666337386632376638
-62316566646539633961353865636165313966663339336436316165323966326561363166613134
-32373764333839313338353162326363373430393031333038646631333836323237643537376462
-33623836396536343335333665366561363737333864363963383836353234633739626466316561
-63346638316365363364316530656563343537326534353137396433646333626666313735366331
-31373465303032306636373437393366316639393065336336306130346234313038316539353037
-36333164306566313539633464373132643234306335633361386637393231306566333832386566
-35356661633535306531623961346635613730653566663536393234373839613961626632313837
-62363062346534623961373266363561326666316161643366386133323163636532363437623266
-38646464366463353162376635313764353338616439633566633862636238643265663465396161
-65333238623833346631653264336430656539623561353135353363326139323234376333346436
-31633365613730663133656532653937373334386335643138663666626230343339663232656336
-36613931623233303164646630363966353730643531356130643265363332386333313132343433
-37653233336337373533313839393365623532376439656537326439663864326639636462613830
-38323832333865613139336632363534616639313566303131326339353934396534336261333839
-63303730363732613037386265663132326264613435666138633639303761623361623836616163
-62663263376231383036663062376333656362303666383962333762653066396339393231636533
-37386538636635366463663434653564656664316230653836646639333736316434356339393435
-39656564333330393436336135656262363862353263613664643063633365336161366664353765
-36356232613234386265396436346130353763636538346636663234633237663133323066316563
-31636237643538376632663462626363386234306334303062343530306161306265633031366161
-63393830656333633864376335623231653230396635616331666236666661643330356135343931
-35356335323332346361666538343065643565333133393137323536363438326563313531336336
-39613330653331356436326437653936386531663037336539643165316131663435363766326435
-37316466666166303262383265653833633437313732363632636235363037326561353032623134
-6239663434363939386230356530333036656637303161626465
+34326635363163333038303363346632613636306133616266343732323036656335643366646264
+3938363837343132633665323362323133663430633165310a303562396164626233653535623336
+34646463376565646435616564616235663836663466353234343030353363626131613134643431
+6535653237343635300a393162633862323261376530396630643539313162653161396438366236
+39633866303562393131636537653932306138643766653632323834373361323938393131656331
+64653335393632336533343135313766643361633739613333666461663962343134636263333333
+30663966306434323331373136366333623262393962363031353564383133306433306261616631
+39323738373163653861653866366139346666333338303435333435663532343466393561616230
+31656234376564366533663762366639363134613666363532336463613863363862353839313034
+32343938656461643531373535363837663336303137323766663966613136313365333734366233
+32613630343034356136313661616532356163336561633562386337613937616535306533623838
+31666363336363653436623635303231366364343137343532613263313436356365393330666638
+65383161613561343361326431623338356338323164656536306162333764346131623235633664
+64666635343765316134653936666137613465363735316562616336636233383439653564316135
+61623466373965323437306537313761353832376462396465306532356162643966643534633666
+35643066653166313335633737393362353630623639336366323161666232353930396434333630
+31353232663837393764653465303133616265636132316430393936323735663136383539336462
+37333262373738366266653532393937326163363832356438373635646465646230623738633232
+61626530323834383838333861363335613034366661343138336638323432306135356363353330
+63396538663731383637333763663763376361313739366266373065303230373135653831643735
+62356365653935386130643364393963353335633539663061633838373132633336613664356631
+65616639643461666538653334666465393965663862343530656265663032653561343833336563
+31653533383665306166393431626161363364346265643631373366316434336234653264666164
+32373336326434666561383463383037633338646635636364366563666464346433643064323032
+66313065303638636635353864613238346537386131303666386264376561393134613438316239
+30623238356663393632326531643732313433383638333866363161656534393134313937383161
+65306439393965353461363439336165356562323262633664653231633538386661616238303732
+37623964613335393330663862666135666664353134303861653232623730626533616335643539
+62396361356465323165366235303362383736386664663935353666613132663762303238346533
+38303665333639323336643466353637636364643631613231613164303664336462353831363662
+33373865326563653632643131313330663237636135376563336565633162613033356163663333
+37383231306333343436366535396463636130353663303830343933623135343661653030643438
+36363663656138326435313565383864373036653832663163633236363961303238346234633231
+33653235643666353266316463373665633661333262303764346466636639316138656266656235
+65353936356230613130373339336631396639303533366239363037626365653262353563643334
+63623537663966353332383838653939653062663864396235633232376635383035313961386638
+33623062336630653432663234303561663233633566343862303631663337383834393930666537
+66376633303034316435366237366464366336313932666337356664323265343533306230343332
+32366239643033333635343563353437633439663839613733636339353933613762303733343736
+65633937653161623732393137313062393636373461306265373461396538663937623263323630
+65626230666636336233303166666664366361366534386466393337373162646262356138636433
+32346238643937343865653165326566346531626238643434623765353836653061623064653166
+62396531333937393363633835663930323138656365313865373733636135333735656138353030
+64313461356232633065613139376134303433613663653733663266376437306337396662353130
+39613732666566636434656466343839353634663736636636666231336235396439393961313366
+65363130666635663633646663656430386538343931346233396563613339333331663930306132
+38363034333434633933303862383965303835343961343562346466393466393165663965343936
+62316234663738356361393836363939393962616639306366653934386539373736636233623763
+30643165353665313235373366366164343461616238313239313737626465653930366466623164
+38653533346335633437653237613436333463373163646261376264376438656131366263353862
+38386361346438343036373761383164666465663436363132373662343266666433383663663333
+31326434666136623865626635663232333766343538383839303435646439386133613663373736
+31373664353630313461363162663866333366613666646337363761333237393635393864373531
+33386434386536343033633664373963323937646535373231623836396334373431353964386566
+31633065346534323566653734663261353866613635316165336534666134653439613463323031
+63656435643132633664393234396230396336326139386632303633393130316566353834376135
+31373663326665333164626433303938666366666463643134356236613738636434626665663461
+66376665363633393530616365643139313436383137323062383763613931353330643634616236
+31323131666536613433396538643364336562366433623437336564663638333136313531623761
+35636431383562393237663533333161333933643662666635623965386435356534633832373531
+35343132663861313931636530666237353166633031366330643731663561346133373831633137
+30633332633362396664333736613630346437353836613237323835313730333033343430323236
+64373663653563343838323438396661363839623261663339333062656264323866386536633439
+39346532633864633663356431663535343664376265376566653861616434313665616264626230
+33316134386630313139343030393435626564353666343734376561616437343032306566303031
+32353663653537666137343831633164303934303436356161313661613164666431653037363539
+65326366323033366663623736626366613239323033356566383334373434313636336230643639
+63646131343636303262626230653633393735323030373531346437396663313162623332316362
+34366239326366633961363236313930303435646135366565626564383663306636623034653465
+62373539663561366435356538386664373664653239313936623362326636353563343337336632
+31333133383562653935656265363136363532653431623830396130636233306563623663333531
+38383664366363306662383532656366356266323031613630336338656362643562373034633933
+61623865316636643430653562623535643966306265613833396266626564326161383666616263
+66663664303431353866613237316539343835366531363166633136633965386532613831346566
+35313334356132626337633339363166303637313665303464343635323163383231636238613066
+34613462386533326638643764346661346361343166376337353136313361656561396238626538
+61666431636661643665323330643239613734663332336638613435653563303835306639316162
+39363432643364393036333334643430663763363234666463323231336135343763653063343533
+32373862383062346261646331376633316463393365303931303535373137663561396636323633
+65626533383337393838323963326361623663386639656264366662326262653161336661306137
+64356561623164303465633562393462396166316233633561323565666433376565646534346132
+34343862393766346534393662316336393363363937313765663237383961356266656233623432
+65383465633830393064393262343133376161646239663166393339643034343635343265636233
+64623664653538343961326663626365333533613338366332396437616466326362346463656465
+30323233343564396238613038663835353538336163333933373538393766633532653736613165
+39343938373535343135656430663263626366346535333833393566363938306430396664623864
+39303539373262383438356566663736623364363766396238323730306263373639303262376463
+63353066306534313031343933343632613634366565386230636137653530393334373832646339
+39396535336466336364666461383639303433383563343236366336316637353032316430646362
+65326339383635333666396233323539316664343031613333653133343732303335633131633031
+66353338363535323734623332633939343230363761646461356534343030326161353131313963
+30323331393133366330653862396265343938623366366164633534653538613461326139353436
+32353939633536616663333763393532323765353533633065373064613438383566373264353362
+37396137353464376362656662303530343261666530663931383031363830356234393162336131
+66313339623064623233393130616532613038623636393035623935346565393061633566663062
+65663563356230316665363863373839326464303632333136643136323334663263343561663530
+33363763393463373637366462653036336461366264333433393366316438343565656232616133
+34333762656562353734383833376234383161396263613534313736346330666237343937313661
+65613631323966393666323834323564356437313032633830616163656365353539623031313762
+65323266626366666366396161373562633938303361396665663536316236333236383234386432
+37666336663362623365343632353734623131346636653539316635336265303137323064313032
+33613036343231666232306233623266663466656362316439643263643163616139303939393430
+63663332626161336637626433386264613131363933313937373030396262343238343565363161
+33666365343534656366366430646639656664656534643831346136643064383931396430383966
+36653166353766656262333434303436643339346365613239386630363430613465366632383733
+31323737616236633535613030313564656364363234386634383234393639313366323333623764
+31353861653964663764633332656133316562373164633433623266623531343663643939633236
+64333635303637653337353164326237316262656237636236643335633331303532353531346531
+64643765353735333634303936356131613866326335376331393733326633653536333563326530
+37353566343236393832653964656262636531376464646433656364353738363762323661646437
+33623234343565646539316361663331623133323238393264613566633930346561613533353862
+38353336623131366331336535626132636638393337376236396462333839363764653264653837
+34326265376538353833343830653431646464643762613661303963363534656465363564366139
+35646461616263646365303232396331343532626635303631313934656332393837616264306234
+37313966656462353161363661386336636363663437346532326361613864353961366432356237
+37386536393866326662343334353237633436383235633636383666613136386465316363393939
+32303138643761653735323037346464653635366430356336313966643537646135623938613033
+65373835303539383830643838383231363735383938373638663165623966356662396665303032
+33646564306334336663636165303633346131373239316564343631306437383462303961626432
+63396263653039336134343530653639356466616331306431633635376364613765663464346433
+34333332663766383838653535643765383761363261326233643832353334386439396263336363
+37336362313062616639663731363038633634383937373034656664626436383735613139393163
+62353933336431356633346166356166616632373035363366393231383232353831633061333833
+39316538636662333936373731363531663562623931643761353566343662363236356231323934
+38343232393932313837323636383763633664643561383936653235303635313532333862633836
+36303865366132316337623165396264613565323937316166653566653738343838663932646463
+31623361303230343037386133343065633633316265633739643137343939663339656165306534
+30346437666261323336613264353231333936633031653235633831396263653139643637663761
+32643436396534643766316364666339613732313132356663613736623333653861376331626663
+65636136303938376531323431323231363662303462353232613963373764616137333832383033
+65633262313662383136646161323231643836313363383333616637353838333361663237373232
+36626661313039613632653261636333303731396232346536666563326465393637383366383130
+30306139383233343965623064353238316138336139363161616234643865366366336135346430
+62393638376539643564343065396539313264396236613032306464346461613832663536373336
+61633336616264353265313336353262646234316338626362653236346565646339663733363230
+37393562383137336636383765363066636363373632613265653837356564313435303932333062
+32393436343733383963336337613662666561336363303632333035346633386339303965333861
+39333839613030326163336566623239323261346239353438303337316162353066343031303363
+37383564316664336432303834653736346539306562663165313464356631663537383761323836
+39363530393461666535306332333632643162663136323337323234353036623835343638333035
+39373464633538393339626363633132343831653730376535623232653662613065326463313464
+39323037643537626638343238343030386336326235376439313934313438653665643238366463
+63393435643638353662333465396331323838313032653736343639373838336664633761323839
+33663563366461313964363465373531386561613331373935363430363935363436643139616365
+66346635333233313464313034643432383763616235326538363464303366636565393736353230
+66356162373862383338346166333030616565643930626261623733626665333135626564623237
+62393766313663366537306261613536356264303063383037626636366465653431383838313963
+38666536613438333935633966643866623737646335323239613666316634613065323134303630
+32313661303735613336373937396532353362306666383664376533643464303332643466383330
+32343765633235356134626132383132306463366564323631323530363337343863316238393930
+39356334303361306535653565653230336433646564353234633736663636333832353838363161
+36623139666432666161313562373232656663646637326562396161633839366133623266356261
+35373536623062306664653633343437653361333031303964353436636330353033653964313738
+38663534376233383739643665303635613132643139346161633031623333653163343762336639
+37363465373366386132393530326163363064383931313231646236313862383562666633366631
+38646537643434653137613765653838383234366538653563363237663262323936646137366664
+36383032623839316165626663623639363466666366373666326133616266663265383365663666
+39316334663862656437303837613638643839343139663765613065323433346138396564376462
+30366138316631343434396532313431313762636330653936366161623561643035356434363936
+61643762613638316634613365623731333831616664356335613764373865623964623138643939
+36623765333933336630666533343462313062623463646335643865356365343535643465373435
+36623461336364373631663733613233303865353230363933333338643861313362613935366663
+61643037326163613435373264653332386337396239393238313864316235363162396466306539
+64643864316230363632313833326136386237366364316436346437643731393930653137373231
+65363637316636303438343465366262353832633538343837386637376235663230336530643836
+39633362313963643134323734313033336433663066316531303331376463653537336463356364
+32316366393464313036666433303031633437653736303935333733373535623732373463643031
+31383031626566623239346337616134666436616465396439343736346662336537326265353264
+39373666383265323233376234333233346331363364633735323266376133306634373735323265
+35636461306361353531663237616239643565633036653230333435646163376433616635393133
+64663266383235666461666531616464373233356132333231313637396663366536666264613364
+30333639636365626338363837623934616331353735343336656235373335616638363462383032
+33396338346231363036613732333466633539393037326664653237643733366665356232336338
+64626265633035386164636534613461636236306563316465333537333364333263323061393330
+36323130376261373339613931363634386163326263303237393931616435666566393466336465
+34396163613731613238613264316430313163666536623337376434393765356438373565626339
+35333164333037626262626635316561323435653432613435383439653364633831616233303530
+66656130313531316661306565313536653133303664303362643361653364383731363039343532
+61396535373630343037376537396431373362643639393633636433326335353230366161656362
+63313933393235386664353761613530636332366332383134353936313639306435356462616639
+62386564363766306334346637353166376361353634366331326638643735373038626333666361
+61623163356532373765633530316635313161346434626538333332613233316630366565346534
+62336436333838303732366536626433353135636362333436613763323730396562616361306665
+35646634623861396232626533333265343761393632393161363063646663663938363535353531
+34636433353237386362313132633732646438643230653438313761386335333731393337346665
+39316239626636323435303932613637373231623337353838313337356632336234623434623038
+66366435376434366364353737656230393531633636633036333630376133313165333963636432
+32353431666532373436316133353439383461353834346439313531333338333764316264343136
+32353733363031376337336666636537613032376361343533323362626132396632633533643163
+66313862623433636438613230646338653961343861623433623864326163363135633864373231
+66313935353164363466356164616363653761623565663032313264656565623864383732376334
+31613538623166663736373535363633623937323261386433386436373361623162626361363033
+35393063663664373230613635353762333238353937633730623861626236663935333134326132
+61343864376639633164333436623563633635343236333664333663653431643664386631376162
+39613766393530313938653562333630343765316461326665386664643134643661666539373131
+35373565313763336136653035656138313162333965663565353531336362616637363830383462
+62343866623838343066653035613031346362303263636436656434303039393434643531666238
+31633363373036356336333235363134616362393362636561316265363366386530666465656531
+37366431373564656533363534613633393739663666666566303538363139643833323537356163
+61396533353536333330343130326663613135393237653438323439623836363162393435646236
+36636631366234663536323463303538303434633632316438343935353162316632663939313437
+36666538323463643462323234626262333131353238333031346139333535656539363336646332
+30353830623536396662313264323637663637353934636532306331323166316535343131336639
+32396237313539653030366164343336623463656261616661376638346561646632623434393166
+62383033313931653235356236363862393837616365616332653730383833376165323735333632
+33303966643462626438303132383233663065353032643362306331663632616535346362643137
+33323736393038356362356135363733326263303430633136383137653734363331623331373537
+63353833336236626664616265383464633335623861353739623863653866323534343163393466
+37666163383465383734643430386437613866616361393561336364346437346164313665363634
+32303539613165613631353239666339336639303561303234336135326137613363656335353761
+37616537353132353561303730326330386435636165303464616232633531613132623636653432
+34353637336338626564353364613962393365333639653133356165343032326430616237396536
+63653033326238336363353061303031393064616163656162376362663061643236643232333266
+62653761383338323837383361383965323963393935626634333661356661396139356566303830
+38313133313564353030643866313366646338376666396435356264373239636666373861363964
+31363863393033633063326237666630666631393036656233336238353736343534633238393532
+62663335393839613137373863346263396361386235346439323437353531626537313965663262
+32636434386238323634616336336464333963633432333932653462666661393933666531303136
+34363432386637323136656335306663656232626631663464396565303465323636326431343762
+66383339336133636431353538643838663331373736636563626537623361363231633934663931
+35366365333036366661363263393062373130383062646332636330326139343266666234323835
+31636463633237373532363333306136396437356236303961623133353630653435396462313264
+34336239373839663061346461313137393333306534646465366430393164646430613964323638
+62666638346130383464633339396364643835323036303039656230343564623663313238326333
+30653364613661306539373832616638636563653963353835343265383865306233356438303464
+62303761363839316237653036316563303466373763323164316331356263656664393831396130
+32636135306166366230353834313330383035383964353031663431613434623331616165613565
+34623765663564636463363431643736613433316366393862353433323032616435303334396230
+38356266623566356637373561343331366665373964373564616138306531356439
--- a/group_vars/linode.yaml
+++ b/group_vars/linode.yaml
@ -19,8 +19,9 @@
 #  - 2600:3c00::c

 dns_servers:
-  - 127.0.0.1
-  - ::1
+  - 8.8.8.8
+  - 1.1.1.1
+  - 9.9.9.9

 timezone: Etc/UTC

--- a/group_vars/monitor_servers/main.yaml
+++ b/group_vars/monitor_servers/main.yaml
@ -8,6 +8,8 @@ alertmanager_web_external_url: https://monitor.kill0.net/alertmanager
 prometheus_web_route_prefix: /
 alertmanager_web_route_prefix: /

+prometheus_file_sd_config_d_files: []
+
 prometheus_config:
  global:
    scrape_interval: 15s
@ -16,6 +18,10 @@ prometheus_config:
      region: dallas
      provider: linode
      replica: A
+  remote_write:
+    - url: http://localhost:9009/api/v1/push
+      headers:
+        X-Scope-OrgID: kill0-net
  alerting:
    alertmanagers:
      - static_configs:
@ -75,10 +81,13 @@ prometheus_config:
            - dns.google
            - vpn-home.kill0.net
            - ping-home.kill0.net
-            - 10.255.0.16
+            - 169.254.0.2
            - vpn1-sch.corp.nmi.com
-            - vpn-chi.ops.nmi.com
-            - vpn-ash.ops.nmi.com
+            - gp-chi.ops.nmi.com
+            - gp-ash.ops.nmi.com
+            - 172.16.100.1
+            - 172.16.100.2
+            - 172.16.10.16
      relabel_configs:
        - source_labels: [__address__]
          target_label: __param_target
@ -169,6 +178,82 @@ prometheus_config:
      static_configs:
        - targets:
            - "localhost:10912"
+    - job_name: grafana
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - "localhost:3002"
+#    - job_name: process-exporter
+#      scrape_interval: 5s
+#      static_configs:
+#        - targets:
+#            - "localhost:9256"
+    - job_name: loki
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - "localhost:3100"
+    - job_name: promtail
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - jump0.kill0.net:9080
+            - mine0.kill0.net:9080
+    - job_name: gitea
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:3001
+    - job_name: karma
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:8080
+    - job_name: kthxbye
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:8081
+    - job_name: smokeping
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:9374
+    - job_name: mimir
+      scrape_interval: 5s
+      static_configs:
+        - targets:
+            - localhost:9009
+    - &snmp_job
+      job_name: snmp
+      static_configs:
+        - targets:
+          - 172.16.100.1
+          - 172.16.100.2
+      metrics_path: /snmp
+      params:
+        auth: [public_v2]
+        module:
+          - if_mib
+          - ip_mib
+      relabel_configs:
+        - source_labels: [__address__]
+          target_label: __param_target
+        - source_labels: [__param_target]
+          target_label: instance
+        - target_label: __address__
+          replacement: 127.0.0.1:9116
+    - job_name: snmp_exporter
+      static_configs:
+        - targets:
+          - localhost:9116
+    - <<: *snmp_job
+      job_name: snmp-long
+      scrape_interval: 30s
+      scrape_timeout: 30s
+      static_configs:
+        - targets: []
+
  rule_files:
    - rules.yaml

@ -220,6 +305,10 @@ prometheus_rules_config:
          expr: up{job=~"thanos.+"} == 0
          labels:
            severity: critical
+        - alert: Down
+          expr: up == 0
+          labels:
+            severity: critical
        - alert: FileSystemUsage
          expr: ((node_filesystem_size_bytes{mountpoint!~"fuse.lxcfs|tmpfs"} - node_filesystem_free_bytes) / node_filesystem_size_bytes) > 0.80
          for: 1m
@ -272,6 +361,13 @@ prometheus_rules_config:
            # summary: Certificates expiring in < 14 days
            summary: "{% raw %}Blackbox SSL certificate will expire soon (instance {{ $labels.instance }}){% endraw %}"
            description: "{% raw %}SSL certificate expires in 14 days\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}{% endraw %}"
+    - name: snmp.rules
+      rules:
+        - alert: PortDown
+          expr: ifAdminStatus{ifName=~"(Gi|eth).+", ifAlias!~".+laptop|notebook.+"} == 1 and ifOperStatus == 2
+          for: 1m
+        - alert: PortFlapping
+          expr: changes(ifOperStatus{ifName=~"(Gi|eth).+"}[5m]) > 2

 blackbox_exporter_config:
  modules:
@ -301,34 +397,6 @@ blackbox_exporter_config:
      http:
        method: GET

-#  route:
-#    receiver: pushover-receiver
-#    mute_time_intervals:
-#      - quiet_hours
-#    routes:
-#      - receiver: blackhole
-#        match:
-#          alertname: MaintenanceMode
-#      #- receiver: blackhole
-#      #  match:
-#      #    alertname: QuietHours
-#  receivers:
-#    - name: blackhole
-#    - name: pushover-receiver
-#      pushover_configs:
-#        - token: "{{ vault_pushover_token }}"
-#          user_key: "{{ vault_pushover_user_key }}"
-#  inhibit_rules:
-#    - source_match:
-#        alertname: MaintenanceMode
-#    #- source_match:
-#    #    alertname: QuietHours
-#  time_intervals:
-#    - name: quiet_hours
-#      times:
-#        - start_time: 03:00
-#          end_time: 15:00
-
 alertmanager_config:
  inhibit_rules:
    - source_match:
@ -337,9 +405,13 @@ alertmanager_config:
    - name: blackhole
    - name: pushover-receiver
      pushover_configs:
-        - token: agwd6wv7xveakykb8e5rz7rw3eg2v3
+        - token: "{{ vault_alertmanager_pushover_token }}"
          user_key: 28G1x3lT4oUtlck50R1H3e6j8kDHjb
+    - name: discord
+      discord_configs:
+        - webhook_url: "{{ vault_alertmanager_discord_webhook_url }}"
  route:
+    repeat_interval: 24h
    receiver: pushover-receiver
    routes:
      - match:
@ -351,6 +423,8 @@ alertmanager_config:
      - receiver: pushover-receiver
        mute_time_intervals:
          - quiet_hours
+        continue: true
+      - receiver: discord
  time_intervals:
    - name: quiet_hours
      time_intervals:
@ -366,7 +440,7 @@ node_exporter_du_directories:
  - /var/lib/loki

 firewall_ipset_loki:
-  - 10.255.0.0/24
+  - 169.254.0.0/24

 karma_config:
  alertmanager:
@ -411,3 +485,112 @@ karma_config:
 thanos_bucket_config: "{{ vault_thanos_bucket_config }}"

 kthxbye_listen: :8081
+
+loki_storage_config:
+  tsdb_shipper:
+    active_index_directory: "{{ loki_var_path }}/tsdb-index"
+    cache_location: "{{ loki_var_path }}/tsdb-cache"
+  gcs:
+    bucket_name: kill0-net-loki
+    service_account: "{{ vault_loki_gcs_service_account | string }}"
+
+loki_schema_config:
+  configs:
+    - from: 2023-08-11
+      index:
+        period: 24h
+        prefix: index_
+      object_store: gcs
+      schema: v12
+      store: tsdb
+    - from: 2024-04-10
+      index:
+        period: 24h
+        prefix: index_
+      object_store: gcs
+      schema: v13
+      store: tsdb
+
+loki_query_scheduler:
+  max_outstanding_requests_per_tenant: 32768
+
+loki_querier:
+  max_concurrent: 16
+
+loki_compactor:
+  working_directory: "{{ loki_var_path }}/retention"
+  delete_request_store: gcs
+  compaction_interval: 10m
+  retention_enabled: true
+  retention_delete_delay: 2h
+  retention_delete_worker_count: 150
+
+loki_ruler:
+  alertmanager_url: http://localhost:9093
+  storage:
+    type: gcs
+    gcs:
+      bucket_name: kill0-net-loki
+      service_account: "{{ vault_loki_gcs_service_account | string }}"
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true
+
+rsyslog_d:
+  - name: loki
+    priority: 10
+    content: |
+      if $hostname == [ "ap0", "coresw0", "fw0", "power0", "172.16.100.1", "172.16.100.2" ] then {
+        action(
+          type="omfwd"
+          target="localhost"
+          port="1514"
+          protocol="tcp"
+          action.resumeretrycount="-1"
+          queue.type="linkedlist"
+          queue.size="1000000"
+          queue.filename="loki-fwd"
+          queue.saveonshutdown="on"
+          keepalive="on"
+          template="RSYSLOG_SyslogProtocol23Format"
+          tcp_framing="octet-counted"
+        )
+      }
+
+smokeping_prober_config:
+  targets:
+    - hosts:
+      - dns.google
+      - vpn-home.kill0.net
+      - ping-home.kill0.net
+      - vpn1-sch.corp.nmi.com
+      - gp-chi.ops.nmi.com
+      - gp-ash.ops.nmi.com
+      - 169.254.0.2
+      - 172.16.100.1
+      - 172.16.100.2
+      - 172.16.10.16
+      network: ip4
+    - hosts:
+      - dns.google
+      - ping-home.kill0.net
+      - fc00::ffff:169.255.0.2
+      - fc00::ffff:169.255.0.16
+      network: ip6
+
+mimir_common:
+  storage:
+    backend: gcs
+    gcs:
+      bucket_name: kill0-net-mimir
+      service_account: "{{ vault_mimir_gcs_service_account | string }}"
+
+mimir_blocks_storage:
+  storage_prefix: blocks
+
+mimir_alertmanager_storage:
+  storage_prefix: alertmanager
+
+mimir_ruler_storage:
+  storage_prefix: ruler
--- a/group_vars/monitor_servers/vault.yaml
+++ b/group_vars/monitor_servers/vault.yaml
@ -0,0 +1,17 @@
+$ANSIBLE_VAULT;1.1;AES256
+35346264373635663161356339313438613932623165613239353162316265333231623434383030
+6435323137313638633663356635373464393730663834320a346362633362323864373636346165
+37363637663037653932313165653333643833376133383336363930623338333134623562353239
+6430363062323865650a363330653031383666386637633333646339393064396330313037363239
+30626538373432633031666264646236613936333965366430653031303131626161376633346435
+63323165366666663362353661353634636339393930343862336132613466636131343861343835
+64633531336139353961626565363434316230393739626531366661653132616566363234393036
+35656331383038396665376236373531323931313632656331356235353664636264393664346131
+38633038303364373166366633646330393636366134626437376662386235626233633831363062
+32636461646661613734353739663934333365313932306363666464656236366634653032303031
+34333032373935343366626537386231306666663934326664353432323338353235306231363464
+64653561663662363064313436653036613038633033623737666335636331656461653535643864
+62376539343761666366333331373164623230663639373231373763653938343535646166303639
+31616463316364366130653033643935356461363938386264306162623933336338363365316162
+63396436316338306136616265643562353931356239393661333161396537653366643765303031
+64323639653263323837
--- a/group_vars/name_servers/main.yaml
+++ b/group_vars/name_servers/main.yaml
@ -1,57 +0,0 @@
---
-nsd_linode_xfr:
-  - "{{ lookup('dig', 'axfr1.linode.com.') }}"
-  - "{{ lookup('dig', 'axfr2.linode.com.') }}"
-  - "{{ lookup('dig', 'axfr3.linode.com.') }}"
-  - "{{ lookup('dig', 'axfr4.linode.com.') }}"
-  - "{{ lookup('dig', 'axfr5.linode.com.') }}"
-  - "{{ lookup('dig', 'axfr1.linode.com./AAAA') }}"
-  - "{{ lookup('dig', 'axfr2.linode.com./AAAA') }}"
-  - "{{ lookup('dig', 'axfr3.linode.com./AAAA') }}"
-  - "{{ lookup('dig', 'axfr4.linode.com./AAAA') }}"
-  - "{{ lookup('dig', 'axfr5.linode.com./AAAA') }}"
-
-nsd_provide_xfr:
-  - "{{ lookup('dig', 'axfr1.linode.com.') }} NOKEY"
-  - "{{ lookup('dig', 'axfr2.linode.com.') }} NOKEY"
-  - "{{ lookup('dig', 'axfr3.linode.com.') }} NOKEY"
-  - "{{ lookup('dig', 'axfr4.linode.com.') }} NOKEY"
-  - "{{ lookup('dig', 'axfr5.linode.com.') }} NOKEY"
-  - "{{ lookup('dig', 'axfr1.linode.com./AAAA') }} NOKEY"
-  - "{{ lookup('dig', 'axfr2.linode.com./AAAA') }} NOKEY"
-  - "{{ lookup('dig', 'axfr3.linode.com./AAAA') }} NOKEY"
-  - "{{ lookup('dig', 'axfr4.linode.com./AAAA') }} NOKEY"
-  - "{{ lookup('dig', 'axfr5.linode.com./AAAA') }} NOKEY"
-
-firewall_dns_whitelist: "{{ nsd_linode_xfr }}"
-firewall_ipset_dns: "{{ nsd_linode_xfr }}"
-
-nsd_zones:
-  - name: cavi.cc
-    zonefile: cavi.cc.zone
-    provide-xfr: "{{ nsd_provide_xfr }}"
-    notify: "{{ nsd_provide_xfr }}"
-  - name: kill0.net
-    zonefile: kill0.net.zone
-    provide-xfr: "{{ nsd_provide_xfr }}"
-    notify: "{{ nsd_provide_xfr }}"
-  - name: kill0.com
-    zonefile: kill0.com.zone
-    provide-xfr: "{{ nsd_provide_xfr }}"
-    notify: "{{ nsd_provide_xfr }}"
-  - name: chill9.com
-    zonefile: chill9.com.zone
-    provide-xfr: "{{ nsd_provide_xfr }}"
-    notify: "{{ nsd_provide_xfr }}"
-  - name: chill9.net
-    zonefile: chill9.net.zone
-    provide-xfr: "{{ nsd_provide_xfr }}"
-    notify: "{{ nsd_provide_xfr }}"
-  - name: confabulator.net
-    zonefile: confabulator.net.zone
-    provide-xfr: "{{ nsd_provide_xfr }}"
-    notify: "{{ nsd_provide_xfr }}"
-  - name: ctrl-v.org
-    zonefile: ctrl-v.org.zone
-    provide-xfr: "{{ nsd_provide_xfr }}"
-    notify: "{{ nsd_provide_xfr }}"
--- a/group_vars/stats_servers/main.yaml
+++ b/group_vars/stats_servers/main.yaml
@ -24,9 +24,8 @@ grafana_config:
    http_port: "{{ grafana_port }}"

 grafana_ssl_enabled: true
-grafana_ssl_certificate: "/etc/letsencrypt/live/{{ grafana_domain }}/fullchain.pem"
-grafana_ssl_certificate_key: "/etc/letsencrypt/live/{{ grafana_domain }}/privkey.pem"
-# grafana_ssl_dhparam: "/etc/letsencrypt/ssl-dhparams.pem"
+grafana_ssl_certificate: "/var/lib/lego/certificates/{{ grafana_domain }}.crt"
+grafana_ssl_certificate_key: "/var/lib/lego/certificates/{{ grafana_domain }}.key"

 grafana_datasources:
  apiVersion: 1
--- a/host_vars/jump0.kill0.net/all.yaml
+++ b/host_vars/jump0.kill0.net/all.yaml
@ -18,25 +18,54 @@ certbot_certificates:
  - domains:
      - cavi.cc
    email: rcavicchioni@gmail.com
+  - domains:
+      - proxy.kill0.net
+    email: rcavicchioni@gmail.com
+
+lego_user_environ:
+  GCE_PROJECT: kill0-net
+  GCE_SERVICE_ACCOUNT_FILE: "{{ lego_etc_dir_path }}/credentials.json"
+
+lego_bin_user_args:
+  - --email rcavicchioni@gmail.com
+  - --dns gcloud
+
+lego_bin_renew_user_args:
+  - --renew-hook "systemctl reload nginx"
+
+lego_domains:
+  - name: cavi.cc
+  - name: dl.kill0.net
+  - name: git.kill0.net
+  - name: monitor.kill0.net
+  - name: proxy.kill0.net
+  - name: stats.kill0.net

 autossh_config: []

 wireguard_interfaces:
  wg0:
-    address: 10.255.0.1/32
+    address:
+      - 169.254.0.1/24
+      - fc00::ffff:169.254.0.1/64
    private_key: "{{ vault_wireguard_private_keys.wg0 }}"
    listen_port: 51820
+    table: 'off'
  wg1:
    address:
      - 192.168.255.1/24
-      - 2600:3c00:e000:343::1/128
+      - fc01::ffff:192.168.255.1/128
+      - 2600:3c00:e000:343::ffff:192.168.255.1/128
    private_key: "{{ vault_wireguard_private_keys.wg1 }}"
    listen_port: 51821

 restic_tidy_enabled: true

+nginx_htpasswd_files: "{{ vault_nginx_htpasswd_files }}"
+
 nginx_vhosts:
  cavicc:
+    server:
      - server_name: cavi.cc
        root: /var/www/cavicc
        listen:
@ -49,7 +78,66 @@ nginx_vhosts:
      - server_name: cavi.cc
        root: /var/www/cavicc
        listen:
-        - 443 ssl http2
-        - "[::]:443 ssl http2"
-      ssl_certificate: /etc/letsencrypt/live/cavi.cc/fullchain.pem
-      ssl_certificate_key: /etc/letsencrypt/live/cavi.cc/privkey.pem
+          - 443 ssl
+          - "[::]:443 ssl"
+        ssl_certificate: /var/lib/lego/certificates/cavi.cc.crt
+        ssl_certificate_key: /var/lib/lego/certificates/cavi.cc.key
+        # ssl_certificate: /etc/letsencrypt/live/cavi.cc/fullchain.pem
+        # ssl_certificate_key: /etc/letsencrypt/live/cavi.cc/privkey.pem
+        raw: |
+          location / {
+            add_header Alt-Svc 'h3=":$server_port"; ma=86400'; 
+          }
+
+  proxy:
+    upstream:
+      - name: loki_backend
+        server:
+          - localhost:3100
+      #- name: prometheus_backend
+      #  server:
+      #    - localhost:9090
+    map:
+      - name: $http_upgrade
+        variable: $connection_upgrade
+        content:
+          default: upgrade
+          '': close
+    server:
+      - server_name: proxy.kill0.net
+        root: /var/empty
+        listen:
+          - 80
+          - "[::]:80"
+        raw: |
+          location / {
+              return 301 https://$server_name$request_uri;
+          }
+      - server_name: proxy.kill0.net
+        root: /var/empty
+        listen:
+          - 443 ssl
+          - "[::]:443 ssl"
+        # ssl_certificate: /etc/letsencrypt/live/proxy.kill0.net/fullchain.pem
+        # ssl_certificate_key: /etc/letsencrypt/live/proxy.kill0.net/privkey.pem
+        ssl_certificate: /var/lib/lego/certificates/proxy.kill0.net.crt
+        ssl_certificate_key: /var/lib/lego/certificates/proxy.kill0.net.key
+        raw: |
+          auth_basic "Proxy";
+          auth_basic_user_file /etc/nginx/proxy.htpasswd;
+
+          location / {
+            add_header Alt-Svc 'h3=":$server_port"; ma=86400'; 
+          }
+
+          location /loki {
+            proxy_http_version 1.1;
+            proxy_pass http://loki_backend;
+            proxy_set_header Connection $connection_upgrade;
+            proxy_set_header Host $http_host;
+            proxy_set_header Upgrade $http_upgrade;
+          }
+
+          location /prometheus/ {
+            proxy_pass http://prometheus_backend/;
+          }
--- a/host_vars/mine0.kill0.net/main.yaml
+++ b/host_vars/mine0.kill0.net/main.yaml
@ -161,6 +161,8 @@ openvpn_certificates:

 wireguard_interfaces:
  wg0:
-    address: 10.255.0.16/32
+    address:
+      - 169.254.0.2/24
+      - fc00::ffff:169.254.0.2/64
    private_key: "{{ vault_wireguard_private_keys.wg0 }}"
    listen_port: 51820
--- a/host_vars/nal-hutta.yaml
+++ b/host_vars/nal-hutta.yaml
@ -1,22 +0,0 @@
---
-#network_interfaces:
-#  - name: eth0
-#    address:
-#      - 45.56.123.101/24
-#      - 2600:3c00::f03c:91ff:fed5:eeec/64
-#    gateway:
-#      - 45.56.123.1
-#      - fe80::1
-
-firewall_allowed_tcp_ports:
-  v4:
-    - 443
-    - 80
-    - 8186
-  v6:
-    - 443
-    - 80
-    - 8186
-
-postfix_sasl_passwd_map:
-  "[smtp.fastmail.com]:465": "foo:bar"
--- a/host_vars/rmq1.yaml
+++ b/host_vars/rmq1.yaml
@ -1,17 +0,0 @@
---
-keepalived_vrrp_instances:
-  VI_1:
-    state: MASTER
-    interface: eth0
-    virtual_router_id: 51
-    priority: 254
-    authentication:
-      auth_type: PASS
-      auth_pass: asdf
-    unicast_peer: |
-      {{ groups['rabbitmq_servers'] | map('extract', hostvars, ['ansible_eth0', 'ipv4', 'address']) | difference([ansible_default_ipv4.address])| list }}
-    virtual_ipaddress:
-      - 10.100.100.20/24
-    track_script:
-      - chk_rabbitmq
-      - chk_amqp_port
--- a/host_vars/rmq2.yaml
+++ b/host_vars/rmq2.yaml
@ -1,17 +0,0 @@
---
-keepalived_vrrp_instances:
-  VI_1:
-    state: BACKUP
-    interface: eth0
-    virtual_router_id: 51
-    priority: 253
-    authentication:
-      auth_type: PASS
-      auth_pass: asdf
-    unicast_peer: |
-      {{ groups['rabbitmq_servers'] | map('extract', hostvars, ['ansible_eth0', 'ipv4', 'address']) | difference([ansible_default_ipv4.address])| list }}
-    virtual_ipaddress:
-      - 10.100.100.20/24
-    track_script:
-      - chk_rabbitmq
-      - chk_amqp_port
--- a/host_vars/rmq3.yaml
+++ b/host_vars/rmq3.yaml
@ -1,17 +0,0 @@
---
-keepalived_vrrp_instances:
-  VI_1:
-    state: BACKUP
-    interface: eth0
-    virtual_router_id: 51
-    priority: 252
-    authentication:
-      auth_type: PASS
-      auth_pass: asdf
-    unicast_peer: |
-      {{ groups['rabbitmq_servers'] | map('extract', hostvars, ['ansible_eth0', 'ipv4', 'address']) | difference([ansible_default_ipv4.address])| list }}
-    virtual_ipaddress:
-      - 10.100.100.20/24
-    track_script:
-      - chk_rabbitmq
-      - chk_amqp_port
--- a/host_vars/ubuntu.yaml
+++ b/host_vars/ubuntu.yaml
@ -1,7 +0,0 @@
---
-#network_interfaces:
-#  - name: enp1s0
-#    address:
-#      - 192.168.124.124/24
-#    gateway4: 192.168.124.1
-#
--- a/inventory.yaml
+++ b/inventory.yaml
@ -21,9 +21,6 @@ all:
    monitor_servers:
      hosts:
        jump0.kill0.net
-    name_servers:
-      hosts:
-        jump0.kill0.net
    linode:
      hosts:
        mine0.kill0.net:
--- a/playbook.yaml
+++ b/playbook.yaml
@ -3,25 +3,59 @@
  become: true
  roles:
    - common
+    - role: network
+      tags:
+        - network
+        - netplan
    - util
    - sudo
    - hostsfile
    - certs
+    - role: rsyslog
+      tags:
        - rsyslog
+        - syslog
+        - logging
    - users
-    - network
    - dns
+    - role: firewall
+      tags:
        - firewall
+        - iptables
    - openssh
+    - role: wireguard
+      tags:
        - wireguard
+        - vpn
    - chrony
    - unattended-upgrades
    - postfix
    - restic
-    - node_exporter
-    - blackbox_exporter
-    - mtail
+    - role: node_exporter
+      tags:
+        - prometheus
+        - monitoring
+    - role: blackbox_exporter
+      tags:
+        - prometheus
+        - monitoring
+    - role: mtail
+      tags:
+        - prometheus
+        - monitoring
    - supervisor
+    # - vector
+    - role: promtail
+      tags:
+        - promtail
+        - loki
+        - logging
+    - role: cloudflared
+      tags:
+        - cloudflared
+        - zerotrust
+        - access
+        - vpn
 - hosts: minecraft_servers
  become: true
  roles:
@ -34,40 +68,98 @@
 - hosts: git_servers
  become: true
  roles:
+    - role: certbot
+      tags:
+        - tls
+    - role: nginx
+      tags:
        - nginx
-    - certbot
+    - role: gitea
+      tags:
        - gitea
+        - git
 - hosts: stats_servers
  become: true
  roles:
+    - role: certbot
+      tags:
+        - tls
+    - role: nginx
+      tags:
        - nginx
-    - certbot
-    - influxdb
+    - role: grafana
+      tags:
        - grafana
+        - monitoring
+        - o11y
 - hosts: monitor_servers
  become: true
  roles:
+    - certbot
+    - role: nginx
+      tags:
        - nginx
    - role: prometheus
      tags:
        - prometheus
        - monitoring
-    - alertmanager
-    - blackbox_exporter
-    - pushgateway
+    - role: alertmanager
+      tags:
+        - prometheus
+        - monitoring
+    - role: blackbox_exporter
+      tags:
+        - prometheus
+        - monitoring
+    - role: pushgateway
+      tags:
+        - prometheus
+        - monitoring
    - role: karma
      tags:
+        - prometheus
        - monitoring
    - role: kthxbye
      tags:
+        - prometheus
        - monitoring
    - role: thanos
      tags:
+        - prometheus
        - thanos
        - monitoring
- hosts: name_servers
-  become: true
-  roles:
-    - nsd
+    - role: loki
+      tags:
+        - loki
+        - logging
+    - role: logcli
+      tags:
+        - logcli
+        - loki
+        - logging
+    - role: smokeping_prober
+      tags:
+        - prometheus
+        - monitoring
+        - smokeping
+    - role: mimir
+      tags:
+        - prometheus
+        - mimir
+        - monitoring
+    - role: snmp_exporter
+      tags:
+        - prometheus
+        - snmp_exporter
+        - monitoring
+    - role: lego
+      tags:
+        - acme
+        - certificates
+        - lego
+        - letsencrypt
+        - pki
+        - tls
+

 # vim:ft=yaml.ansible:
--- a/roles/certbot/defaults/main.yaml
+++ b/roles/certbot/defaults/main.yaml
@ -1,22 +1,35 @@
 ---
 certbot_package_name: certbot
-certbot_package_state: present
+certbot_package_state: latest
+
+certbot_plugins:
+  - certbot-dns-cloudflare
+  - certbot-dns-digitalocean
+  - certbot-dns-dnsimple
+  - certbot-dns-dnsmadeeasy
+  - certbot-dns-gehirn
+  - certbot-dns-google
+  - certbot-dns-linode
+  - certbot-dns-luadns
+  - certbot-dns-nsone
+  - certbot-dns-ovh
+  - certbot-dns-rfc2136
+  - certbot-dns-route53
+  - certbot-dns-sakuracloud

 certbot_service_name: certbot.service

+certbot_bin_path: /usr/local/bin
+certbot_path: "{{ certbot_bin_path }}/certbot"
+
 certbot_timer_name: certbot.timer
 certbot_timer_state: started
-certbot_timer_enabled: yes
+certbot_timer_enabled: true

-certbot_cron_state: present
-certbot_cron_user: root
-certbot_cron_file_path: /etc/cron.d/certbot
-certbot_cron_env:
-  path: /usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-  shell: /bin/sh
-certbot_cron_command: test -x /usr/bin/certbot -a \! -d /run/systemd/system && perl -e 'sleep int(rand(43200))' && certbot -q renew
-certbot_cron_hour: "*/12"
-certbot_cron_minute: "0"
+certbot_etc_path: /etc/letsencrypt
+certbot_live_path: "{{ certbot_etc_path }}/live"

 certbot_system_timer_on_calender: "*-*-* 00,12:00:00"
 certbot_system_timer_randomized_delay_sec: 43200
+
+certbot_credential_path: /root/.secrets/certbot
--- a/roles/certbot/handlers/main.yaml
+++ b/roles/certbot/handlers/main.yaml
@ -1,6 +1,4 @@
 ---
 - name: systemd daemon-reload
-  systemd:
-    name: "{{ certbot_service_name }}"
-    daemon_reload: yes
-    state: restarted
+  ansible.builtin.systemd:
+    daemon_reload: true
--- a/roles/certbot/tasks/Ubuntu.yaml
+++ b/roles/certbot/tasks/Ubuntu.yaml
@ -1,4 +0,0 @@
---
- name: configure ppa
-  apt_repository:
-    repo: "ppa:certbot/certbot"
--- a/roles/certbot/tasks/configure-linode.yaml
+++ b/roles/certbot/tasks/configure-linode.yaml
@ -0,0 +1,23 @@
+---
+- name: configure linode credentials
+  ansible.builtin.copy:
+    dest: "{{ certbot_credential_path }}/linode.ini"
+    owner: root
+    group: root
+    mode: 0600
+    content: "{{ certbot_dns_linode_credentials }}"
+  no_log: true
+
+- name: certbot (linode)
+  ansible.builtin.shell: >
+    certbot certonly \
+      --dns-linode \
+      --dns-linode-credentials "{{ certbot_credential_path }}/linode.ini" \
+      --quiet \
+      --agree-tos \
+      --noninteractive \
+      --email "{{ item.email }}" \
+      --domain "{{ item.domains | join(',') }}"
+  args:
+    creates: "{{ certbot_live_path }}/{{ item.domains | first }}/cert.pem"
+  loop: "{{ certbot_certificates | default([]) }}"
--- a/roles/certbot/tasks/default.yaml
+++ b/roles/certbot/tasks/default.yaml
--- a/roles/certbot/tasks/issue.yaml
+++ b/roles/certbot/tasks/issue.yaml
@ -1,9 +1 @@
 ---
- name: "determine if certificate for {{ item.domains | join(', ') }}" 
-  stat:
-    path: "/etc/letsencrypt/live/{{ item.domains | first }}/cert.pem"
-  register: st
-
- name: "request certificate for {{ item.domains | join(', ') }}"
-  command: "certbot certonly -q --webroot -w {{ certbot_challenge_webroot_path }} --agree-tos --noninteractive --email {{ item.email }} -d {{ item.domains | join(',') }}"
-  when: not st.stat.exists
--- a/roles/certbot/tasks/main.yaml
+++ b/roles/certbot/tasks/main.yaml
@ -23,65 +23,51 @@
      paths:
        - tasks

- name: install certbot modules
-  package:
+- name: install certbot
+  ansible.builtin.pip:
    name: "{{ certbot_package_name }}"
    state: "{{ certbot_package_state }}"

- name: configure challenge webroot
-  file:
-    path: "{{ certbot_challenge_webroot_path }}"
-    state: "directory"
+- name: install certbot plugins
+  ansible.builtin.pip:
+    name: "{{ certbot_plugins }}"
+    state: latest
+
+- name: create credential path
+  ansible.builtin.file:
+    path: "{{ certbot_credential_path }}"
    owner: root
    group: root
-    mode: 0755
+    mode: 0700
+    state: directory

 - name: request certificates
  ansible.builtin.include_tasks: "issue.yaml"
  loop: "{{ certbot_certificates }}"

- name: configure systemd timer
-  block:
-    - name: create systemd timer override directory
-      file:
-        path: "/etc/systemd/system/{{ certbot_timer_name }}.d"
-        owner: root
-        group: root
-        mode: 0755
-        state: directory
+- name: include linode tasks
+  ansible.builtin.include_tasks: configure-linode.yaml

-    - name: configure systemd timer options
-      template:
-        src: certbot.timer.j2
-        dest: "/etc/systemd/system/{{ certbot_timer_name }}.d/override.conf"
+- name: configure renewal service
+  ansible.builtin.template:
+    src: certbot.service.j2
+    dest: "/etc/systemd/system/certbot.service"
    owner: root
    group: root
    mode: 0644
  notify: systemd daemon-reload
-    - name: enable the timer
-      systemd:
-        name: "{{ certbot_timer_name }}"
-        state: "{{ certbot_timer_state }}"
-        enabled: "{{ certbot_timer_enabled }}"
-  when: ansible_service_mgr == "systemd"

- name: configure cron job
-  block:
-    - name: configure env
-      cron:
-        name: "{{ item.key | upper }}"
-        env: yes
-        job: "{{ item.value }}"
-        user: "{{ certbot_cron_user }}"
-        cron_file: "{{ certbot_cron_file_path }}"
-        state: "{{ certbot_cron_state }}"
-      loop: "{{ certbot_cron_env | dict2items }}"
-    - name: create job
-      cron:
-        name: certbot
-        user: "{{ certbot_cron_user }}"
-        hour: "{{ certbot_cron_hour }}"
-        minute: "{{ certbot_cron_minute }}"
-        cron_file: "{{ certbot_cron_file_path }}"
-        job: "{{ certbot_cron_command }}"
-        state: "{{ certbot_cron_state }}"
+- name: configure renewal timer
+  ansible.builtin.template:
+    src: certbot.timer.j2
+    dest: "/etc/systemd/system/certbot.timer"
+    owner: root
+    group: root
+    mode: 0644
+  notify: systemd daemon-reload
+
+- name: manage timer
+  ansible.builtin.systemd:
+    name: "{{ certbot_timer_name }}"
+    enabled: "{{ certbot_timer_enabled }}"
+    state: "{{ certbot_timer_state }}"
--- a/roles/certbot/templates/certbot.service.j2
+++ b/roles/certbot/templates/certbot.service.j2
@ -0,0 +1,14 @@
+# {{ ansible_managed }}
+
+[Unit]
+Description=Certbot renewal
+After=network-online.target
+Wants=network-online.target
+Wants={{ certbot_timer_name }}
+
+[Service]
+Type=oneshot
+ExecStart={{ certbot_path }} --quiet renew
+
+[Install]
+WantedBy=multi-user.target
--- a/roles/certbot/templates/certbot.timer.j2
+++ b/roles/certbot/templates/certbot.timer.j2
@ -1,5 +1,12 @@
 # {{ ansible_managed }}

+[Unit]
+Description=Certbot renewal
+Requires={{ certbot_service_name }}
+
 [Timer]
 OnCalendar={{ certbot_system_timer_on_calender }} 
 RandomizedDelaySec={{ certbot_system_timer_randomized_delay_sec }}
+
+[Install]
+WantedBy=timers.target
--- a/roles/cloudflared/defaults/main.yaml
+++ b/roles/cloudflared/defaults/main.yaml
@ -0,0 +1,10 @@
+---
+cloudflared_package_name: cloudflared
+cloudflared_package_state: present
+
+cloudflared_service_name: cloudflared.service
+cloudflared_service_enabled: true
+cloudflared_service_state: started
+
+cloudflared_apt_repository_repo: "deb [signed-by=/etc/apt/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared {{ ansible_lsb.codename }} main"
+cloudflared_apt_repository_state: present
--- a/roles/cloudflared/files/cloudflare-main.gpg
+++ b/roles/cloudflared/files/cloudflare-main.gpg
--- a/roles/cloudflared/tasks/Debian.yaml
+++ b/roles/cloudflared/tasks/Debian.yaml
@ -0,0 +1,14 @@
+---
+- name: trust cloudflare apt respository key
+  ansible.builtin.copy:
+    src: "cloudflare-main.gpg"
+    dest: "/etc/apt/keyrings/cloudflare-main.gpg"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: configure cloudflare apt repository
+  ansible.builtin.apt_repository:
+    repo: "{{ cloudflared_apt_repository_repo }}"
+    state: "{{ cloudflared_apt_repository_state | default('present') }}"
+    filename: cloudflared
--- a/roles/cloudflared/tasks/install.yaml
+++ b/roles/cloudflared/tasks/install.yaml
@ -0,0 +1,5 @@
+---
+- name: install package
+  ansible.builtin.package:
+    name: "{{ cloudflared_package_name }}"
+    state: "{{ cloudflared_package_state | default('present') }}"
--- a/roles/cloudflared/tasks/main.yaml
+++ b/roles/cloudflared/tasks/main.yaml
@ -0,0 +1,28 @@
+---
+- name: gather os specific variables
+  ansible.builtin.include_vars: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: include os specific tasks
+  ansible.builtin.include_tasks: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- ansible.builtin.include_tasks: install.yaml
+
+# - ansible.builtin.include_tasks: configure.yaml
--- a/roles/cloudflared/vars/default.yaml
+++ b/roles/cloudflared/vars/default.yaml
--- a/roles/common/defaults/main.yaml
+++ b/roles/common/defaults/main.yaml
@ -1,6 +1,6 @@
 ---
-cron_service_name: cron
-
-timezone: UTC
-
-# vim:ft=yaml.ansible:
+# common_cron_service_name: cron.service
+# common_timezone: Etc/UTC
+# common_locale: C.UTF-8
+# common_apt_update_cache: true
+# common_apt_cache_valid_time: 3600
--- a/roles/common/handlers/main.yaml
+++ b/roles/common/handlers/main.yaml
@ -1,8 +1,5 @@
 ---
 - name: restart cron
-  service:
-    name: "{{ cron_service_name }}"
+  ansible.builtin.service:
+    name: "{{ common_cron_service_name | default('cron.service') }}"
    state: restarted
-  when: cron_service_name is defined
-
-# vim:ft=yaml.ansible:
--- a/roles/common/tasks/Debian.yaml
+++ b/roles/common/tasks/Debian.yaml
@ -1,6 +1,6 @@
 ---
 - name: run apt-get update
-  apt:
-    update_cache: yes
-    cache_valid_time: 3600
+  ansible.builtin.apt:
+    update_cache: "{{ common_apt_update_cache | default(true) }}"
+    cache_valid_time: "{{ common_apt_cache_valid_time | default(3600) }}"
  changed_when: false
--- a/roles/common/tasks/main.yaml
+++ b/roles/common/tasks/main.yaml
@ -24,12 +24,17 @@
        - tasks

 - name: set hostname
-  hostname:
-    name: "{{ hostname | default(inventory_hostname) }}"
+  ansible.builtin.hostname:
+    name: "{{ common_hostname | default(inventory_hostname) }}"

 - name: configure system timezone
-  timezone:
-    name: "{{ timezone }}"
+  ansible.builtin.timezone:
+    name: "{{ common_timezone | default('Etc/UTC') }}"
  notify: restart cron

+- name: configure system locale
+  ansible.builtin.command:
+    cmd: "localectl set-locale {{ common_locale | default('C.UTF-8') }}"
+  when: ansible_facts.env.LANG != (common_locale | default('C.UTF-8'))
+
 # vim:ft=yaml.ansible:
--- a/roles/dl/defaults/main.yaml
+++ b/roles/dl/defaults/main.yaml
@ -4,5 +4,5 @@ dl_server_root: /var/www/dl
 dl_access_log: /var/log/nginx/dl.access.log
 dl_error_log: /var/log/nginx/dl.error.log
 dl_ssl_enabled: false
-dl_ssl_certificate: "/etc/letsencrypt/live/{{ dl_server_name }}/fullchain.pem"
-dl_ssl_certificate_key: "/etc/letsencrypt/live/{{ dl_server_name }}/privkey.pem"
+dl_ssl_certificate: "/var/lib/lego/certificates/{{ dl_server_name }}.crt"
+dl_ssl_certificate_key: "/var/lib/lego/certificates/{{ dl_server_name }}.key"
--- a/roles/dl/templates/nginx.conf.j2
+++ b/roles/dl/templates/nginx.conf.j2
@ -26,10 +26,13 @@ server {
 {% if dl_ssl_enabled is defined and
      dl_ssl_enabled %}
 server {
-    listen 443 ssl http2;
+    listen 443 ssl;
 {% if ansible_all_ipv6_addresses | length %}
-    listen [::]:443 ssl http2;
+    listen [::]:443 ssl;
 {% endif %}
+
+    http2 on;
+
    server_name {{ dl_server_name }};
    access_log {{ dl_access_log }} main;
    error_log {{ dl_error_log }} warn;
@ -46,6 +49,10 @@ server {
    ssl_dhparam {{ dl_ssl_dhparam }};
 {% endif %}

+    location / {
+        add_header Alt-Svc 'h3=":$server_port"; ma=86400';
+    }
+
    location ~ ^\/~(.+?)(\/.*)?$ {
        alias /home/$1/public_html$2;
        index index.html index.htm;
--- a/roles/docker/handlers/main.yaml
+++ b/roles/docker/handlers/main.yaml
@ -0,0 +1,12 @@
+---
+- name: reload docker
+  ansible.builtin.service:
+    name: "{{ docker_service_name | default('docker') }}"
+    state: reloaded
+
+- name: restart docker
+  ansible.builtin.service:
+    name: "{{ docker_service_name | default('docker') }}"
+    state: restarted
+  listen:
+    - restart nftables
--- a/roles/firewall/templates/ip6tables.j2
+++ b/roles/firewall/templates/ip6tables.j2
@ -130,6 +130,9 @@
 {% endif %}
 {% if firewall_ipset_syslog is defined %}
 -A INPUT -p tcp -m tcp --dport 514 -m set --match-set syslog6 src -m comment --comment "accept syslog 514/tcp6" -j LOG_ACCEPT
+-A INPUT -p udp -m udp --dport 514 -m set --match-set syslog6 src -m comment --comment "accept syslog 514/udp6" -j LOG_ACCEPT
+-A INPUT -p tcp -m tcp --dport 1514 -m set --match-set syslog6 src -m comment --comment "accept syslog 1514/tcp6" -j LOG_ACCEPT
+-A INPUT -p udp -m udp --dport 1514 -m set --match-set syslog6 src -m comment --comment "accept syslog 1514/udp6" -j LOG_ACCEPT
 {% endif %}
 {% if firewall_ipset_influxdb is defined %}
 -A INPUT -p tcp -m tcp --dport 8086 -m set --match-set influxdb6 src -m comment --comment "accept influxdb 8086/tcp6" -j LOG_ACCEPT
--- a/roles/firewall/templates/iptables.j2
+++ b/roles/firewall/templates/iptables.j2
@ -117,6 +117,8 @@
 {% if firewall_ipset_syslog is defined %}
 -A INPUT -p tcp -m tcp --dport 514 -m set --match-set syslog4 src -m comment --comment "accept syslog 514/tcp" -j LOG_ACCEPT
 -A INPUT -p udp -m udp --dport 514 -m set --match-set syslog4 src -m comment --comment "accept syslog 514/udp" -j LOG_ACCEPT
+-A INPUT -p tcp -m tcp --dport 1514 -m set --match-set syslog4 src -m comment --comment "accept syslog 1514/tcp" -j LOG_ACCEPT
+-A INPUT -p udp -m udp --dport 1514 -m set --match-set syslog4 src -m comment --comment "accept syslog 1514/udp" -j LOG_ACCEPT
 {% endif %}
 {% if firewall_ipset_influxdb is defined %}
 -A INPUT -p tcp -m tcp --dport 8086 -m set --match-set influxdb4 src -m comment --comment "accept influxdb 8086/tcp" -j LOG_ACCEPT
--- a/roles/gitea/defaults/main.yaml
+++ b/roles/gitea/defaults/main.yaml
@ -53,6 +53,8 @@ gitea_config:
    colorize: no
  service:
    register_manual_confirm: true
+  metrics:
+    enabled: true

 gitea_var_tree:
  - "{{ gitea_var_path }}"
@ -62,6 +64,6 @@ gitea_var_tree:
  - "{{ gitea_var_path }}/backup"

 gitea_ssl_enabled: yes
-gitea_ssl_certificate: "/etc/letsencrypt/live/{{ gitea_domain }}/fullchain.pem"
-gitea_ssl_certificate_key: "/etc/letsencrypt/live/{{ gitea_domain }}/privkey.pem"
+gitea_ssl_certificate: "/var/lib/lego/certificates/{{ gitea_domain }}.crt"
+gitea_ssl_certificate_key: "/var/lib/lego/certificates/{{ gitea_domain }}.key"
 #gitea_ssl_dhparam: "/etc/letsencrypt/ssl-dhparams.pem"
--- a/roles/gitea/templates/nginx.conf.j2
+++ b/roles/gitea/templates/nginx.conf.j2
@ -37,10 +37,13 @@ server {
 {% if gitea_ssl_enabled is defined and
      gitea_ssl_enabled %}
 server {
-    listen 443 ssl http2;
+    listen 443 ssl;
 {% if ansible_all_ipv6_addresses | length %}
-    listen [::]:443 ssl http2;
+    listen [::]:443 ssl;
 {% endif %}
+
+    http2 on;
+
    server_name {{ gitea_domain }};

    access_log /var/log/nginx/gitea.access.log main;
@ -62,6 +65,7 @@ server {
    }

    location / {
+        add_header Alt-Svc 'h3=":$server_port"; ma=86400';
        limit_req zone=req_bad_actors burst=10 nodelay;
        proxy_pass http://gitea_backend;
    }
--- a/roles/grafana/defaults/main.yaml
+++ b/roles/grafana/defaults/main.yaml
@ -26,8 +26,8 @@ grafana_config:
    http_port: "{{ grafana_port }}"

 grafana_ssl_enabled: true
-grafana_ssl_certificate: "/etc/letsencrypt/live/{{ grafana_domain }}/fullchain.pem"
-grafana_ssl_certificate_key: "/etc/letsencrypt/live/{{ grafana_domain }}/privkey.pem"
+grafana_ssl_certificate: "/var/lib/lego/certificates/{{ grafana_domain }}.crt"
+grafana_ssl_certificate_key: "/var/lib/lego/certificates/{{ grafana_domain }}.key"
 # grafana_ssl_dhparam: "/etc/letsencrypt/ssl-dhparams.pem"


--- a/roles/grafana/tasks/default.yaml
+++ b/roles/grafana/tasks/default.yaml
--- a/roles/grafana/templates/nginx.conf.j2
+++ b/roles/grafana/templates/nginx.conf.j2
@ -6,6 +6,11 @@ upstream grafana_backend {
    server 127.0.0.1:{{ grafana_port }};
 }

+map $http_upgrade $connection_upgrade {
+    default upgrade;
+    ''      close;
+}
+
 server {
    listen 80;
 {% if ansible_all_ipv6_addresses | length %}
@ -32,10 +37,13 @@ server {
 {% if grafana_ssl_enabled is defined and
      grafana_ssl_enabled %}
 server {
-    listen 443 ssl http2;
+    listen 443 ssl;
 {% if ansible_all_ipv6_addresses | length %}
-    listen [::]:443 ssl http2;
+    listen [::]:443 ssl;
 {% endif %}
+
+    http2 on;
+
    server_name {{ grafana_domain }};

    access_log /var/log/nginx/grafana.access.log main;
@ -59,7 +67,12 @@ server {
    }

    location / {
+        add_header Alt-Svc 'h3=":$server_port"; ma=86400';
        limit_req zone=req_bad_actors burst=10 nodelay;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection $connection_upgrade;
+        proxy_set_header Host $http_host;
        proxy_pass http://grafana_backend;
    }
 }
--- a/roles/lego/defaults/main.yaml
+++ b/roles/lego/defaults/main.yaml
@ -0,0 +1,130 @@
+---
+lego_go_arch_map:
+  i386: '386'
+  x86_64: 'amd64'
+
+lego_go_arch: "{{ lego_go_arch_map[ansible_architecture] | default('amd64') }}"
+
+lego_version: 4.16.1
+# curl -L -s https://github.com/go-acme/lego/releases/download/v4.14.2/lego_4.14.2_checksums.txt | awk '{ printf "%s: sha256:%s\n", $2, $1 }' | sort
+lego_checksums:
+  lego_v4.16.1_darwin_amd64.tar.gz: sha256:2555ae9c3976bb6d3d783819c7012572fecbd309330a5010dd1f9882332fa349
+  lego_v4.16.1_darwin_arm64.tar.gz: sha256:609789c72a9c8e7f4f5916aa08440a299f63c75fee14f42e61904cda01f0736f
+  lego_v4.16.1_freebsd_386.tar.gz: sha256:41408e99b9f1fb823e53d53feb15cd0cb929ad3cd093b9010c7af7ba71077e55
+  lego_v4.16.1_freebsd_amd64.tar.gz: sha256:9353c009c4801d7646b3c99803a77aa0f2a041f802c8794d16ba4b31af4a8dfb
+  lego_v4.16.1_freebsd_arm64.tar.gz: sha256:c39a98c8401a0fe506ac206ae5ef5e167d1dcd9e7f6bb27def954089c0f99839
+  lego_v4.16.1_freebsd_armv5.tar.gz: sha256:b96b88a84aa51e77da8d4b92f6920b1890ae47c53e59c477d7b3b556b1273446
+  lego_v4.16.1_freebsd_armv6.tar.gz: sha256:ea41ff383adcf98ff70a65e6da49c7c82d16071f3057e44e1c41b2fe34543f19
+  lego_v4.16.1_freebsd_armv7.tar.gz: sha256:6e883cb6c12a7bb703018e85623bf2c548eebfd01047bda75820264bb8ff85f2
+  lego_v4.16.1_linux_386.tar.gz: sha256:3eb2e75cc474b0a0b9a990ddd9c70e7c9631a150487d8434e03a295cfd4b0caa
+  lego_v4.16.1_linux_amd64.tar.gz: sha256:e9826f955337c1fd825d21b073168692711985e25db013ff6b00e9a55a9644b4
+  lego_v4.16.1_linux_arm64.tar.gz: sha256:0669037c2bcff11d0599765c63f186dfc98397b6a827f5cb2e48e9e69c12626c
+  lego_v4.16.1_linux_armv5.tar.gz: sha256:33ff82f3aff43825b0fca7f173825c6cc6b02d9e5607dec147ba172e62c883c9
+  lego_v4.16.1_linux_armv6.tar.gz: sha256:3532a986667fe4ba42366fe09a5487c273c168779f803d878b4cc990d29c5c94
+  lego_v4.16.1_linux_armv7.tar.gz: sha256:b9727c1282a320c22d9fbdbdb59e35810c8b7f94d1382bfa87d564429a89629e
+  lego_v4.16.1_linux_mips64_hardfloat.tar.gz: sha256:055914fab0e26432590fccb54e400e1c0b1ad8d9932f0d418ed9ee7857765eed
+  lego_v4.16.1_linux_mips64_softfloat.tar.gz: sha256:6d79cde9f3f7598276e9f82d2c0fe94b541b35112c0d03797cae4bd9de289d78
+  lego_v4.16.1_linux_mips64le_hardfloat.tar.gz: sha256:5a2421aed70c009d746eff8ffb8a1429dbfdda9c60d08790b53b88d7d4e0b270
+  lego_v4.16.1_linux_mips64le_softfloat.tar.gz: sha256:c1e8afedc29d18e7cb6da4d42c77d41b11041f58637e453be1ac70f65dfba0bc
+  lego_v4.16.1_linux_mips_hardfloat.tar.gz: sha256:07bcd8f03dda24e7db4ef0be065680a8db2d1ec7b217aea2c4ee7f6a6d731928
+  lego_v4.16.1_linux_mips_softfloat.tar.gz: sha256:0367bd328a9355b0191ae0f1b77a20e6a7f6c84a0a65d0a7e4a5f240e7737ed4
+  lego_v4.16.1_linux_mipsle_hardfloat.tar.gz: sha256:49c6117c24e351921e9fdfc0fa01dc7dd007001602b4743f2854b85dde7dd410
+  lego_v4.16.1_linux_mipsle_softfloat.tar.gz: sha256:e5771a43504deab162291c957c1cf549e287c15f645712c08e56f08e5ed97d4c
+  lego_v4.16.1_openbsd_386.tar.gz: sha256:7aaa14b081b8c2d18717c463b6ecea434c963366c82ad9824bcf61750b130c73
+  lego_v4.16.1_openbsd_amd64.tar.gz: sha256:4249afea73a1f8cdec964a0471e841103d6575f6d8549005ec2c06efa063d0fe
+  lego_v4.16.1_openbsd_arm64.tar.gz: sha256:4e94b6714bfed91c06e7365da1da36624126b323dc2c0fdabe7fd3fb155f7cb5
+  lego_v4.16.1_solaris_amd64.tar.gz: sha256:e9d33547a2671636bf02148677bd790996fb94688b0a055393675c645de150ec
+  lego_v4.16.1_windows_386.zip: sha256:980e5d8e6afb700f28c9b9ab539141c45fbd556e12c5b3deb114d7db056d7f0f
+  lego_v4.16.1_windows_amd64.zip: sha256:2716e8cc14facd60d804f849c1aeff6bb31bfa09719905d8f65ec801ead628ca
+  lego_v4.16.1_windows_arm64.zip: sha256:28179af7c79f01e8347dcaab65fba5b70abd36dcd0a2bcc2d6803cb177f2b72c
+  lego_v4.16.1_windows_armv5.zip: sha256:4017c2f1cbd8c838377e6816daccabc96d063b44749407c68e985af7f04fff6c
+  lego_v4.16.1_windows_armv6.zip: sha256:099992c58012440f693206ab0ea23dd1794f4093fd2ad62b744d6a08e3749efd
+  lego_v4.16.1_windows_armv7.zip: sha256:4b9557137c5d24996c3b44c223edf9495f0ea7df7f9a2d5da5f3dbc8f8ec8b50
+
+lego_github_rel_path: go-acme/lego
+lego_github_project_url: "https://github.com/{{ lego_github_rel_path }}"
+lego_release_file: "lego_v{{ lego_version }}_{{ ansible_system | lower }}_{{ lego_go_arch }}.tar.gz"
+lego_release_url: "{{ lego_github_project_url }}/releases/download/v{{ lego_version }}/{{ lego_release_file }}"
+lego_download_path: "/tmp/{{ lego_release_file }}"
+
+lego_opt_dir_path: "/opt/lego-{{ lego_version }}"
+
+lego_unarchive_dest_path: /tmp/
+lego_extracted_path: "/tmp"
+lego_binaries:
+ - lego
+
+lego_user_name: lego
+lego_user_shell: /usr/sbin/nologin
+lego_user_home: "{{ lego_var_dir_path }}"
+lego_group_name: lego
+
+lego_bin_dir_path: /usr/local/bin
+lego_bin_path: "{{ lego_bin_dir_path }}/lego"
+
+lego_etc_dir_path: /etc/lego
+lego_etc_dir_path_owner: "{{ lego_user_name }}"
+lego_etc_dir_path_group: "{{ lego_group_name }}"
+lego_etc_dir_path_mode: ugo=rx
+lego_etc_dir_path_state: directory
+
+lego_var_dir_path: /var/lib/lego
+lego_var_dir_path_owner: "{{ lego_user_name }}"
+lego_var_dir_path_group: "{{ lego_group_name }}"
+lego_var_dir_path_mode: u=rwx,go=rx
+lego_var_dir_path_state: directory
+
+lego_bin_args:
+  - --accept-tos
+  - --domains %i
+  - --domains www.%i
+
+lego_environ:
+  LEGO_PATH: "{{ lego_var_dir_path }}"
+
+lego_bin_user_args: []
+lego_user_environ: {}
+lego_credential_files: []
+
+lego_service_name: lego@.service
+lego_service_enabled: true
+lego_service_state: started
+
+lego_timer_name: lego@.timer
+lego_timer_enabled: true
+lego_timer_state: started
+
+lego_service_template_src: "{{ lego_service_name }}.j2"
+lego_service_template_dest: "/etc/systemd/system/{{ lego_service_name }}"
+lego_service_template_owner: root
+lego_service_template_group: root
+lego_service_template_mode: ugo=r
+
+lego_timer_template_src: "{{ lego_timer_name }}.j2"
+lego_timer_template_dest: "/etc/systemd/system/{{ lego_timer_name }}"
+lego_timer_template_owner: root
+lego_timer_template_group: root
+lego_timer_template_mode: ugo=r
+
+lego_systemd_service_d_dir_path: /etc/systemd/system/lego@.service.d
+lego_systemd_service_d_dir_path_owner: root
+lego_systemd_service_d_dir_path_group: root
+lego_systemd_service_d_dir_path_mode: ugo=rx
+lego_systemd_service_d_dir_path_state: directory
+
+lego_systemd_service_d_template_src: "environ.conf.j2"
+lego_systemd_service_d_template_dest: "{{ lego_systemd_service_d_dir_path }}/environ.conf"
+lego_systemd_service_d_template_path_owner: root
+lego_systemd_service_d_template_path_group: root
+lego_systemd_service_d_template_path_mode: u=r,go=
+
+lego_credential_file_owner: "{{ lego_user_name }}"
+lego_credential_file_group: "{{ lego_group_name }}"
+lego_credential_file_mode: u=r,go=
+
+# lego_domains:
+#   - name: example.com
+#     # not required
+#     enabled: true
+#     # not required
+#     state: started
--- a/roles/lego/handlers/main.yaml
+++ b/roles/lego/handlers/main.yaml
@ -0,0 +1,5 @@
+---
+- name: restart lego
+  systemd:
+    name: "{{ lego_service_name }}"
+    daemon_reload: true
--- a/roles/lego/tasks/configure.yaml
+++ b/roles/lego/tasks/configure.yaml
@ -0,0 +1,98 @@
+---
+- name: create group
+  ansible.builtin.group:
+    name: "{{ lego_group_name }}"
+    system: true
+
+- name: create user
+  ansible.builtin.user:
+    name: "{{ lego_user_name }}"
+    shell: "{{ lego_user_shell }}"
+    home: "{{ lego_user_home }}"
+    system: true
+    group: "{{ lego_group_name }}"
+
+- name: create var path
+  ansible.builtin.file:
+    path: "{{ lego_var_dir_path }}"
+    owner: "{{ lego_var_dir_path_owner }}"
+    group: "{{ lego_var_dir_path_group }}"
+    mode: "{{ lego_var_dir_path_mode }}"
+    state: "{{ lego_var_dir_path_state }}"
+
+- name: create etc path
+  ansible.builtin.file:
+    path: "{{ lego_etc_dir_path }}"
+    owner: "{{ lego_etc_dir_path_owner }}"
+    group: "{{ lego_etc_dir_path_group }}"
+    mode: "{{ lego_etc_dir_path_mode }}"
+    state: "{{ lego_etc_dir_path_state }}"
+
+- name: "create {{ lego_systemd_service_d_dir_path }}"
+  ansible.builtin.file:
+    path: "{{ lego_systemd_service_d_dir_path }}"
+    owner: "{{ lego_systemd_service_d_dir_path_owner }}"
+    group: "{{ lego_systemd_service_d_dir_path_group }}"
+    mode: "{{ lego_systemd_service_d_dir_path_mode }}"
+    state: "{{ lego_systemd_service_d_dir_path_state }}"
+
+- name: "create {{ lego_systemd_service_d_template_dest }}"
+  ansible.builtin.template:
+    src: "{{ lego_systemd_service_d_template_src }}"
+    dest: "{{ lego_systemd_service_d_template_dest }}"
+    owner: "{{ lego_systemd_service_d_template_path_owner }}"
+    group: "{{ lego_systemd_service_d_template_path_group }}"
+    mode: "{{ lego_systemd_service_d_template_path_mode }}"
+  notify:
+    - restart lego
+
+- name: create credential files
+  ansible.builtin.copy:
+    dest: "{{ lego_etc_dir_path }}/{{ item.name }}"
+    owner: "{{ item.owner | default(lego_credential_file_owner) }}"
+    group: "{{ item.group | default(lego_credential_file_group) }}"
+    mode: "{{ item.mode | default(lego_credential_file_mode) }}"
+    content: "{{ item.content }}"
+  loop: "{{ lego_credential_files | default([]) }}"
+  no_log: true
+
+#- name: configure
+#  ansible.builtin.template:
+#    src: "{{ lego_config_file_template_src }}"
+#    dest: "{{ lego_config_file_template_dest }}"
+#    owner: "{{ lego_config_file_template_owner }}"
+#    group: "{{ lego_config_file_template_group }}"
+#    mode: "{{ lego_config_file_template_mode }}"
+#  notify:
+#    - restart lego
+#
+- name: configure systemd unit
+  ansible.builtin.template:
+    src: "{{ lego_service_template_src }}"
+    dest: "{{ lego_service_template_dest }}"
+    owner: "{{ lego_service_template_owner }}"
+    group: "{{ lego_service_template_group }}"
+    mode: "{{ lego_service_template_mode }}"
+  notify:
+    - restart lego
+
+- name: configure timer
+  ansible.builtin.template:
+    src: "{{ lego_timer_template_src }}"
+    dest: "{{ lego_timer_template_dest }}"
+    owner: "{{ lego_timer_template_owner }}"
+    group: "{{ lego_timer_template_group }}"
+    mode: "{{ lego_timer_template_mode }}"
+#
+#- name: manage service
+#  ansible.builtin.service:
+#    name: "{{ lego_service_name }}"
+#    enabled: "{{ lego_service_enabled | default(true) }}"
+#    state: "{{ lego_service_state | default('started') }}"
+
+- name: manage timers
+  ansible.builtin.systemd:
+    name: "lego@{{ item.name }}.timer"
+    enabled: "{{ item.enabled | default(true) }}"
+    state: "{{ item.state | default('started') }}"
+  loop: "{{ lego_domains | default([]) }}"
--- a/roles/lego/tasks/default.yaml
+++ b/roles/lego/tasks/default.yaml
--- a/roles/lego/tasks/install.yaml
+++ b/roles/lego/tasks/install.yaml
@ -0,0 +1,56 @@
+---
+- name: determine install status
+  ansible.builtin.stat:
+    path: "{{ lego_opt_dir_path }}/lego"
+  register: st
+
+- name: create opt path
+  ansible.builtin.file:
+    path: "{{ lego_opt_dir_path }}"
+    owner: root
+    group: root
+    mode: 0755
+    state: directory
+    
+- block:
+  - name: download
+    ansible.builtin.get_url:
+      url: "{{ lego_release_url }}"
+      dest: "{{ lego_download_path }}"
+      checksum: "{{ lego_checksums[lego_release_file] }}"
+    register: dl
+    until: dl is success
+    retries: 5
+    delay: 10
+
+  - name: extract
+    ansible.builtin.unarchive:
+      src: "{{ lego_download_path }}"
+      dest: "{{ lego_unarchive_dest_path }}"
+      remote_src: true
+
+  - name: install
+    ansible.builtin.copy:
+      src: "{{ lego_extracted_path }}/{{ item }}"
+      dest: "{{ lego_opt_dir_path }}/{{ item }}"
+      remote_src: true
+    loop: "{{ lego_binaries }}"
+  when: not st.stat.exists
+
+- name: permissions
+  ansible.builtin.file:
+    path: "{{ lego_opt_dir_path }}/{{ item }}"
+    owner: root
+    group: root
+    mode: 0755
+  loop: "{{ lego_binaries }}"
+
+- name: symlink
+  ansible.builtin.file:
+    src: "{{ lego_opt_dir_path }}/{{ item }}"
+    dest: "/usr/local/bin/{{ item }}"
+    owner: root
+    group: root
+    mode: 0755
+    state: link
+  loop: "{{ lego_binaries }}"
--- a/roles/lego/tasks/main.yaml
+++ b/roles/lego/tasks/main.yaml
@ -0,0 +1,28 @@
+---
+- name: gather os specific variables
+  ansible.builtin.include_vars: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: include os specific tasks
+  ansible.builtin.include_tasks: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- ansible.builtin.include_tasks: install.yaml
+
+- ansible.builtin.include_tasks: configure.yaml
--- a/roles/lego/templates/environ.conf.j2
+++ b/roles/lego/templates/environ.conf.j2
@ -0,0 +1,8 @@
+# {{ ansible_managed }}
+
+[Service]
+{% if lego_user_environ is defined %}
+{% for k, v in lego_user_environ.items() %}
+Environment={{ k | upper }}={{ v }}
+{% endfor %}
+{% endif %}
--- a/roles/lego/templates/lego@.service.j2
+++ b/roles/lego/templates/lego@.service.j2
@ -0,0 +1,31 @@
+# {{ ansible_managed }}
+
+[Unit]
+Description=Let's Encrypt client and ACME library written in Go
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+{% if lego_environ is defined %}
+{% for k, v in lego_environ.items() %}
+Environment={{ k | upper }}={{ v }}
+{% endfor %}
+{% endif %}
+Type=oneshot
+User={{ lego_user_name }}
+ExecStart={{ lego_bin_path }} \
+{% for arg in lego_bin_args | default([]) + lego_bin_user_args | default([]) %}
+  {{ arg }} \
+{% endfor %}
+  renew \
+{% for arg in lego_bin_renew_user_args | default([]) %}
+  {{ arg }} {% if not loop.last %}\{{ "\n"}}{% endif %}
+{% if loop.last %}
+
+{% endif %}
+{% endfor %}
+
+WorkingDirectory={{ lego_var_dir_path }}
+
+[Install]
+WantedBy=multi-user.target
--- a/roles/lego/templates/lego@.timer.j2
+++ b/roles/lego/templates/lego@.timer.j2
@ -0,0 +1,11 @@
+[Unit]
+Description=Certbot renewal
+Description=Let's Encrypt client and ACME library written in Go
+Requires={{ lego_service_name }}%i
+
+[Timer]
+OnCalendar=*-*-* 00,12:00:00
+# RandomizedDelaySec=1
+
+[Install]
+WantedBy=timers.target
--- a/roles/lego/vars/default.yaml
+++ b/roles/lego/vars/default.yaml
--- a/roles/logcli/defaults/main.yaml
+++ b/roles/logcli/defaults/main.yaml
@ -0,0 +1,57 @@
+---
+logcli_go_arch_map:
+  i386: '386'
+  x86_64: 'amd64'
+
+logcli_go_arch: "{{ logcli_go_arch_map[ansible_architecture] | default('amd64') }}"
+
+logcli_version: 2.9.5
+logcli_checksums:
+  logcli-darwin-amd64.zip: sha256:b224dc8872167be0c5f07b1c22471b21604419b625b4a6e69b2c7751bc409d98
+  logcli-darwin-arm64.zip: sha256:ad93156ae1132038de7a6b42633bdc59aac1a04e816aeae2796bc6dddddff14a
+  logcli-freebsd-amd64.zip: sha256:952f48394a080b88a100001b9c454e8793071ba4cd8cc95811bd446b4215a9a3
+  logcli-linux-amd64.zip: sha256:a22f7e29bb9ef8f6f70e31926bbffc646b9e36b3265458e199c497b305d21cc2
+  logcli-linux-arm.zip: sha256:0ad5c86191916121acea30d44011d84d33e5ca27497691980de16f1508b209f5
+  logcli-linux-arm64.zip: sha256:06b6a6b961f5004c51eb7922509dbbb189701b1f3925ba1bb2289894fef7861e
+  logcli-windows-amd64.exe.zip: sha256:d1a37c56fa2a1dfa97855d2a26826ba89569d50846a6022be03936423f04e19b
+  loki-canary-darwin-amd64.zip: sha256:9f73e81666397e195ae092c518df32200bab71f72ff778c839abba0283f8f4b3
+  loki-canary-darwin-arm64.zip: sha256:fa3a96bec9b30ec06bf5271182646161ab8056c51b07e00da14ce21d53bbd871
+  loki-canary-freebsd-amd64.zip: sha256:170c0ea9bf6349cce9b9fb5be6b27d0b8477fc57e5a0849ad7c828ba3de79f15
+  loki-canary-linux-amd64.zip: sha256:e4ff7cfb302851b98d4df1dc7793b3fdc7fd9680d2e75fc0484abcd08412f198
+  loki-canary-linux-arm.zip: sha256:02750db39ecba743da3036ca28a3b426c7d068efeee86b875f7870ba8798dca2
+  loki-canary-linux-arm64.zip: sha256:e0c0c31c89cad8ddffbd11f9467778e9b30bdfbdce955fba67871365a07ab3a1
+  loki-canary-windows-amd64.exe.zip: sha256:54564cbd123fbdd1b95fe9882bd916e2e9432b53826a97c04179c48ff0314912
+  loki-darwin-amd64.zip: sha256:b5831c0da363b3b075ddbdaa6e6e1323858b17c0d6c0052908aebaa637bc522c
+  loki-darwin-arm64.zip: sha256:1b73e4867730c252ce0e3720dd42fea5bd7921dd3cda4aa5f3764e43e1495374
+  loki-freebsd-amd64.zip: sha256:c3ac9b0aa16ca494a1537c28fe036440cd701d5273c5c8bbdb47426ecb5a041e
+  loki-linux-amd64.zip: sha256:9d919a55e7a2dbaeab46e777a0589d7e304c71fed011f989143883cbc887e348
+  loki-linux-arm.zip: sha256:104efc28b322523bf5bced67bdcc3746e1f7f872057f6ef54f25ab00ce426b39
+  loki-linux-arm64.zip: sha256:491833bf201c55388b82c3d1f583a9d4426c1b778ed3dc710cd67c8cbbbb67bb
+  loki-windows-amd64.exe.zip: sha256:1acee64bb69bd54ff6549edd2f670d0a3802727d9efced8705c7a712412d8ef7
+  promtail-darwin-amd64.zip: sha256:54032f2781d3acfef7dd7ad12b7f38ec4f5d0eb8ba047ebecb9911a6dd4b6cc6
+  promtail-darwin-arm64.zip: sha256:405ed21efcaa21ae5bbe4b7e16ca888ae8238716c46a176ea9c5e2a7b2b2a633
+  promtail-freebsd-amd64.zip: sha256:5a68f6fa6c7ae96919f13b4fffb188f72f9b16e38f40cf3962b97989c9739a99
+  promtail-linux-amd64.zip: sha256:e444bcff2d6677d284350819d3d1b7b473a1699357689230254fbc602b28dac7
+  promtail-linux-arm.zip: sha256:d0cc7552b8ce69534893040e6518288a6899c4f3acf9d4e7d32335f5f2f6145d
+  promtail-linux-arm64.zip: sha256:b23bd750dc5f6a76d808826ebc9d3c8b3540adb329578b650571a10d2be348b8
+  promtail-windows-386.exe.zip: sha256:a121de0b043db194c65422f863211efe566da3bec338a92f0623dff6f3c435d1
+  promtail-windows-amd64.exe.zip: sha256:d9c4b5bb58d3ece2e4ff78cd7fef65f5fadd7d9fe73ceb2dfa4a2990f944466f
+
+logcli_github_rel_path: grafana/loki
+logcli_github_project_url: "https://github.com/{{ logcli_github_rel_path }}"
+logcli_release_file: "logcli-{{ ansible_system | lower }}-{{ logcli_go_arch }}.zip"
+logcli_release_url: "{{ logcli_github_project_url }}/releases/download/v{{ logcli_version }}/{{ logcli_release_file }}"
+logcli_download_path: "/tmp/logcli-{{ logcli_version }}-{{ ansible_system | lower }}-{{ logcli_go_arch }}.zip"
+
+logcli_opt_path: "/opt/logcli-{{ logcli_version }}"
+
+logcli_unarchive_dest_path: /tmp/
+logcli_extracted_path: "/tmp/logcli-{{ ansible_system | lower }}-{{ logcli_go_arch }}"
+logcli_binaries:
+ - logcli
+
+logcli_loki_addr: http://localhost:3100
+
+logcli_profile_d_path: /etc/profile.d/logcli.sh
+logcli_profile_d_env:
+  LOKI_ADDR: "{{ logcli_loki_addr }}"
--- a/roles/logcli/tasks/configure.yaml
+++ b/roles/logcli/tasks/configure.yaml
@ -0,0 +1,11 @@
+---
+- name: set logcli environment variables
+  ansible.builtin.copy:
+    dest: "{{ logcli_profile_d_path }}"
+    owner: root
+    group: root
+    mode: 0755
+    content: |
+      {% for k, v in logcli_profile_d_env.items() %}
+      export {{ k }}="{{ v }}"
+      {% endfor %}
--- a/roles/logcli/tasks/default.yaml
+++ b/roles/logcli/tasks/default.yaml
--- a/roles/logcli/tasks/install.yaml
+++ b/roles/logcli/tasks/install.yaml
@ -0,0 +1,56 @@
+---
+- name: determine install status
+  ansible.builtin.stat:
+    path: "{{ logcli_opt_path }}/logcli"
+  register: st
+
+- name: create opt path
+  ansible.builtin.file:
+    path: "{{ logcli_opt_path }}"
+    owner: root
+    group: root
+    mode: 0755
+    state: directory
+    
+- block:
+  - name: download
+    ansible.builtin.get_url:
+      url: "{{ logcli_release_url }}"
+      dest: "{{ logcli_download_path }}"
+      checksum: "{{ logcli_checksums[logcli_release_file] }}"
+    register: dl
+    until: dl is success
+    retries: 5
+    delay: 10
+
+  - name: extract
+    ansible.builtin.unarchive:
+      src: "{{ logcli_download_path }}"
+      dest: "{{ logcli_unarchive_dest_path }}"
+      remote_src: true
+
+  - name: install
+    ansible.builtin.copy:
+      src: "{{ logcli_extracted_path }}"
+      dest: "{{ logcli_opt_path }}/{{ item }}"
+      remote_src: true
+    loop: "{{ logcli_binaries }}"
+  when: not st.stat.exists
+
+- name: permissions
+  ansible.builtin.file:
+    path: "{{ logcli_opt_path }}/{{ item }}"
+    owner: root
+    group: root
+    mode: 0755
+  loop: "{{ logcli_binaries }}"
+
+- name: symlink
+  ansible.builtin.file:
+    src: "{{ logcli_opt_path }}/{{ item }}"
+    dest: "/usr/local/bin/{{ item }}"
+    owner: root
+    group: root
+    mode: 0755
+    state: link
+  loop: "{{ logcli_binaries }}"
--- a/roles/logcli/tasks/main.yaml
+++ b/roles/logcli/tasks/main.yaml
@ -0,0 +1,28 @@
+---
+- name: gather os specific variables
+  ansible.builtin.include_vars: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: include os specific tasks
+  ansible.builtin.include_tasks: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- ansible.builtin.include_tasks: install.yaml
+
+- ansible.builtin.include_tasks: configure.yaml
--- a/roles/logcli/vars/default.yaml
+++ b/roles/logcli/vars/default.yaml
--- a/roles/loki/defaults/main.yaml
+++ b/roles/loki/defaults/main.yaml
@ -26,12 +26,17 @@ loki_user_shell: /usr/sbin/nologin
 loki_group: loki
 loki_group_state: "{{ loki_user_state | default('present') }}"

-loki_config_path: /etc/loki.yaml
-
 loki_var_path: /var/lib/loki
 loki_var_owner: "{{ loki_user }}"
 loki_var_group: "{{ loki_group }}"
-loki_var_mode: "0755"
+loki_var_mode: "0700"
+
+loki_etc_path: /etc/loki
+loki_etc_owner: "{{ loki_user }}"
+loki_etc_group: "{{ loki_group }}"
+loki_etc_mode: "0755"
+
+loki_config_path: "{{ loki_etc_path }}/config.yaml"

 loki_bin_path: /usr/local/bin

@ -39,36 +44,51 @@ loki_auth_enabled: false

 loki_server:
  http_listen_port: 3100
+  grpc_listen_port: 9096

-loki_ingester:
-  lifecycler:
-    address: 127.0.0.1
+loki_common:
+  instance_addr: 127.0.0.1
+  path_prefix: "{{ loki_var_path }}"
+  storage:
+    filesystem:
+      chunks_directory: "{{ loki_var_path }}/chunks"
+      rules_directory: "{{ loki_var_path }}/rules"
+  replication_factor: 1
  ring:
    kvstore:
      store: inmemory
-      replication_factor: 1
-    final_sleep: 0s
-  chunk_idle_period: 5m
-  chunk_retain_period: 30s
+
+loki_query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+# loki_storage_config:
+#   {}

 loki_schema_config:
  configs:
-  - from: 2020-05-15
-    store: boltdb
-    object_store: filesystem
+  - from: 2020-10-24
+    store: boltdb-shipper
+    object_store: gcs
    schema: v11
    index:
      prefix: index_
-      period: 168h
+      period: 24h

-loki_storage_config:
-  boltdb:
-    directory: "{{ loki_var_path }}/index"
-  filesystem:
-    directory: "{{ loki_var_path }}/chunks"
+loki_ruler:
+  alertmanager_url: http://localhost:9093
+
+# loki_query_scheduler:
+#   {}
+
+# loki_querier:
+#   {}
+
+# loki_compactor:
+#   {}

 loki_limits_config:
-  enforce_metric_name: false
-  reject_old_samples: true
-  reject_old_samples_max_age: 168h
-  ingestion_burst_size_mb: 16
+  retention_period: 744h
--- a/roles/loki/tasks/configure.yaml
+++ b/roles/loki/tasks/configure.yaml
@ -15,14 +15,13 @@
    home: "{{ loki_var_path }}"
    state: "{{ loki_user_state | default('present') }}"

- name: configure
-  template:
-    src: loki.yaml.j2
-    dest: "{{ loki_config_path }}"
-    owner: root
-    group: root
-    mode: 0444
-  notify: restart loki
+- name: create etc path
+  file:
+    path: "{{ loki_etc_path }}"
+    state: directory
+    owner: "{{ loki_etc_owner }}"
+    group: "{{ loki_etc_group }}"
+    mode: "{{ loki_etc_mode }}"

 - name: create var path
  file:
@ -32,6 +31,15 @@
    group: "{{ loki_var_group }}"
    mode: "{{ loki_var_mode }}"

+- name: configure
+  template:
+    src: config.yaml.j2
+    dest: "{{ loki_config_path }}"
+    owner: "{{ loki_user }}"
+    group: "{{ loki_group }}"
+    mode: 0400
+  notify: restart loki
+
 - name: configure systemd template
  template:
    src: "{{ loki_service_name }}.j2"
--- a/roles/loki/templates/config.yaml.j2
+++ b/roles/loki/templates/config.yaml.j2
@ -0,0 +1,55 @@
+{{ ansible_managed | comment }}
+---
+{% if loki_auth_enabled is defined %}
+auth_enabled: {{ loki_auth_enabled | bool | lower }}
+{% endif %}
+
+{% if loki_server is defined %}
+server:
+  {{ loki_server | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_common is defined %}
+common:
+  {{ loki_common | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_query_range is defined %}
+query_range:
+  {{ loki_query_range | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_storage_config is defined %}
+storage_config:
+  {{ loki_storage_config | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_schema_config is defined %}
+schema_config:
+  {{ loki_schema_config | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_ruler is defined %}
+ruler:
+  {{ loki_ruler | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_query_scheduler is defined %}
+query_scheduler:
+  {{ loki_query_scheduler | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_querier is defined %}
+querier:
+  {{ loki_querier | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_compactor is defined %}
+compactor:
+  {{ loki_compactor | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
+
+{% if loki_limits_config is defined %}
+limits_config:
+  {{ loki_limits_config | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif -%}
--- a/roles/loki/templates/loki.service.j2
+++ b/roles/loki/templates/loki.service.j2
@ -1,19 +1,19 @@
 {{ ansible_managed | comment }}

 [Unit]
-Description=Loki
-After=network-online.target
+Description=Loki service
+After=network.target

 [Service]
 Type=simple
 User={{ loki_user }}
-Group={{ loki_group }}
 ExecStart={{ loki_bin_path }}/loki \
  -config.file {{ loki_config_path }}
-WorkingDirectory={{ loki_var_path }}

-Restart=always
-RestartSec=1
+WorkingDirectory={{ loki_var_path }}
+TimeoutSec = 120
+Restart = on-failure
+RestartSec = 2

 [Install]
 WantedBy=multi-user.target
--- a/roles/loki/templates/loki.yaml.j2
+++ b/roles/loki/templates/loki.yaml.j2
@ -1,30 +0,0 @@
-{{ ansible_managed | comment }}
---
-{% if loki_auth_enabled is defined %}
-auth_enabled: {{ loki_auth_enabled | bool | lower }}
-{% endif %}
-
-{% if loki_server is defined %}
-server:
-  {{ loki_server | to_nice_yaml(indent=2) | indent(2, False) }}
-{% endif -%}
-
-{% if loki_ingester is defined %}
-ingester:
-  {{ loki_ingester | to_nice_yaml(indent=2) | indent(2, False) }}
-{% endif -%}
-
-{% if loki_schema_config is defined %}
-schema_config:
-  {{ loki_schema_config | to_nice_yaml(indent=2) | indent(2, False) }}
-{% endif -%}
-
-{% if loki_storage_config is defined %}
-storage_config:
-  {{ loki_storage_config | to_nice_yaml(indent=2) | indent(2, False) }}
-{% endif -%}
-
-{% if loki_limits_config is defined %}
-limits_config:
-  {{ loki_limits_config | to_nice_yaml(indent=2) | indent(2, False) }}
-{% endif -%}
--- a/roles/mimir/defaults/main.yaml
+++ b/roles/mimir/defaults/main.yaml
@ -0,0 +1,68 @@
+---
+mimir_package_name: mimir
+mimir_package_state: present
+
+mimir_service_name: mimir.service
+mimir_service_enabled: true
+mimir_service_state: started
+
+mimir_apt_repository_repo: deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main
+mimir_apt_repository_state: present
+
+mimir_version_regex: ^mimir, version ([\d.]+)
+
+mimir_user_name: mimir
+mimir_user_comment: mimir user
+mimir_user_state: present
+mimir_user_shell: /usr/sbin/nologin
+mimir_user_system: true
+mimir_user_createhome: false
+
+mimir_group_name: mimir
+mimir_group_state: "{{ mimir_user_state | default('present') }}"
+mimir_group_system: true
+
+mimir_var_dir_path: /var/lib/mimir
+mimir_var_dir_owner: "{{ mimir_user_name }}"
+mimir_var_dir_group: "{{ mimir_group_name }}"
+mimir_var_dir_mode: "0700"
+
+mimir_etc_dir_path: /etc/mimir
+mimir_etc_dir_owner: "{{ mimir_user_name }}"
+mimir_etc_dir_group: "{{ mimir_group_name }}"
+mimir_etc_dir_mode: "0755"
+
+mimir_config_file_path: "{{ mimir_etc_dir_path }}/config.yml"
+mimir_config_file_path_owner: "{{ mimir_user_name }}"
+mimir_config_file_path_group: "{{ mimir_group_name }}"
+mimir_config_file_path_mode: "0755"
+
+m#  imir_common:
+ #   {}
+
+mimir_server:
+  http_listen_port: 9009
+
+mimir_alertmanager:
+  sharding_ring:
+    replication_factor: 1
+
+# mimir_compactor:
+#   {}
+
+#  mimir_distributor:
+#    {}
+
+mimir_ingester:
+  ring:
+    replication_factor: 1
+
+mimir_store_gateway:
+  sharding_ring:
+    replication_factor: 1
+
+#  mimir_blocks_storage:
+#    {}
+
+#  mimir_ruler_storage:
+#    {}
--- a/roles/mimir/files/grafana.gpg
+++ b/roles/mimir/files/grafana.gpg
--- a/roles/mimir/handlers/main.yaml
+++ b/roles/mimir/handlers/main.yaml
@ -0,0 +1,6 @@
+---
+- name: restart mimir
+  systemd:
+    name: "{{ mimir_service_name }}"
+    daemon_reload: true
+    state: restarted
--- a/roles/mimir/tasks/Debian.yaml
+++ b/roles/mimir/tasks/Debian.yaml
@ -0,0 +1,14 @@
+---
+- name: trust grafana apt respository key
+  ansible.builtin.copy:
+    src: "grafana.gpg"
+    dest: "/etc/apt/keyrings/grafana.gpg"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: configure grafana apt repository
+  ansible.builtin.apt_repository:
+    repo: "{{ mimir_apt_repository_repo }}"
+    state: "{{ mimir_apt_repository_state | default('present') }}"
+    filename: grafana
--- a/roles/mimir/tasks/configure.yaml
+++ b/roles/mimir/tasks/configure.yaml
@ -0,0 +1,48 @@
+---
+- name: create group
+  ansible.builtin.group:
+    name: "{{ mimir_group_name }}"
+    system: "{{ mimir_group_system | default(true) }}"
+    state: "{{ mimir_group_name_state | default('present') }}"
+
+- name: create user
+  ansible.builtin.user:
+    name: "{{ mimir_user_name }}"
+    comment: "{{ mimir_user_comment  }}"
+    system: "{{ mimir_user_system | default(true) }}"
+    shell: "{{ mimir_user_shell | default('/usr/sbin/nologin') }}"
+    group: "{{ mimir_group_name }}"
+    createhome: "{{ mimir_user_createhome | default(false) }}"
+    home: "{{ mimir_var_dir_path }}"
+    state: "{{ mimir_user_state | default('present') }}"
+
+- name: create etc path
+  ansible.builtin.file:
+    path: "{{ mimir_etc_dir_path }}"
+    state: directory
+    owner: "{{ mimir_etc_dir_owner }}"
+    group: "{{ mimir_etc_dir_group }}"
+    mode: "{{ mimir_etc_dir_mode }}"
+
+- name: create var path
+  ansible.builtin.file:
+    path: "{{ mimir_var_dir_path }}"
+    state: directory
+    owner: "{{ mimir_var_dir_owner }}"
+    group: "{{ mimir_var_dir_group }}"
+    mode: "{{ mimir_var_dir_mode }}"
+
+- name: configure
+  template:
+    src: config.yml.j2
+    dest: "{{ mimir_config_file_path }}"
+    owner: "{{ mimir_user_name }}"
+    group: "{{ mimir_group_name }}"
+    mode: 0400
+  notify: restart mimir
+
+- name: manage service
+  service:
+    name: "{{ mimir_service_name }}"
+    enabled: "{{ mimir_service_enabled }}"
+    state: "{{ mimir_service_state }}"
--- a/roles/mimir/tasks/default.yaml
+++ b/roles/mimir/tasks/default.yaml
--- a/roles/mimir/tasks/install.yaml
+++ b/roles/mimir/tasks/install.yaml
@ -0,0 +1,5 @@
+---
+- name: install package
+  ansible.builtin.package:
+    name: "{{ mimir_package_name }}"
+    state: "{{ mimir_package_state | default('present') }}"
--- a/roles/mimir/tasks/main.yaml
+++ b/roles/mimir/tasks/main.yaml
@ -0,0 +1,28 @@
+---
+- name: gather os specific variables
+  ansible.builtin.include_vars: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: include os specific tasks
+  ansible.builtin.include_tasks: "{{ lookup('first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- ansible.builtin.include_tasks: install.yaml
+
+- ansible.builtin.include_tasks: configure.yaml
--- a/roles/mimir/templates/config.yml.j2
+++ b/roles/mimir/templates/config.yml.j2
@ -0,0 +1,51 @@
+# {{ ansible_managed }}
+---
+{% if mimir_common is defined %}
+common:
+  {{ mimir_common | to_nice_yaml(indent=2) | indent(4, False) }}
+{% endif -%}
+
+{% if mimir_server is defined %}
+server:
+  {{ mimir_server | to_nice_yaml(indent=2) | indent(4, False) }}
+{% endif -%}
+
+{% if mimir_alertmanager is defined %}
+alertmanager:
+  {{ mimir_alertmanager | to_nice_yaml(indent=2) | indent(4, False) }}
+{% endif -%}
+
+{% if mimir_compactor is defined %}
+compactor:
+  {{ mimir_compactor | to_nice_yaml(indent=2) | indent(4, False) }}
+{% endif -%}
+
+{% if mimir_distributor is defined %}
+distributor:
+  {{ mimir_distributor | to_nice_yaml(indent=2) | indent(4, False) }}
+{% endif -%}
+
+{% if mimir_ingester is defined %}
+ingester:
+  {{ mimir_ingester | to_nice_yaml(indent=2) | indent(4, False) }}
+{% endif -%}
+
+{% if mimir_store_gateway is defined %}
+store_gateway:
+  {{ mimir_store_gateway | to_nice_yaml(indent=2) | indent(4, False) }}
+{% endif -%}
+
+{% if mimir_alertmanager_storage is defined %}
+alertmanager_storage:
+  {{ mimir_alertmanager_storage | to_nice_yaml(indent=2) | indent(4, False) }}
+{% endif -%}
+
+{% if mimir_blocks_storage is defined %}
+blocks_storage:
+  {{ mimir_blocks_storage | to_nice_yaml(indent=2) | indent(4, False) }}
+{% endif -%}
+
+{% if mimir_ruler_storage is defined %}
+ruler_storage:
+  {{ mimir_ruler_storage | to_nice_yaml(indent=2) | indent(4, False) }}
+{% endif -%}
--- a/roles/mimir/vars/default.yaml
+++ b/roles/mimir/vars/default.yaml
--- a/roles/minecraft/defaults/main.yaml
+++ b/roles/minecraft/defaults/main.yaml
@ -11,8 +11,8 @@ minecraft_port: 25565
 minecraft_user: minecraft
 minecraft_group: minecraft

-minecraft_jar_url: https://launcher.mojang.com/v1/objects/e00c4052dac1d59a1188b2aa9d5a87113aaf1122/server.jar
-minecraft_jar_checksum: sha256:deefd056f0cf89c3d7fd48d03f56a8a73943586e8c061fdabd0fd92d32ced2b2
+minecraft_jar_url: https://piston-data.mojang.com/v1/objects/84194a2f286ef7c14ed7ce0090dba59902951553/server.jar
+minecraft_jar_checksum: sha256:3af73a9dc5a102e38147946360dd27d4d70bae7055bf91cf2151cd5d121b79e0

 minecraft_opt_path: /opt/minecraft
 minecraft_var_path: /var/opt/minecraft
--- a/roles/mtail/defaults/main.yaml
+++ b/roles/mtail/defaults/main.yaml
@ -12,7 +12,7 @@ mtail_service_enabled: yes
 mtail_version_regex: ^mtail version (\S+)

 mtail_github_project_url: https://github.com/google/mtail
-mtail_release_file: "mtail_{{ mtail_version }}_{{ ansible_system | capitalize }}_{{ ansible_architecture }}.tar.gz"
+mtail_release_file: "mtail_{{ mtail_version }}_{{ ansible_system | lower }}_{{ mtail_go_arch }}.tar.gz"
 mtail_release_url: "{{ mtail_github_project_url }}/releases/download/v{{ mtail_version }}/{{ mtail_release_file }}"
 mtail_download_path: "/tmp/{{ mtail_release_file }}"
 mtail_checksum_url: "{{ mtail_github_project_url }}/releases/download/v{{ mtail_version }}/checksums.txt"
--- a/roles/mtail/tasks/pre.yaml
+++ b/roles/mtail/tasks/pre.yaml
@ -1,42 +1,4 @@
 ---
-#- name: determine if installed
-#  stat:
-#    path: "{{ mtail_bin_path }}/mtail"
-#  register: st
-#
-#- name: set mtail_installed
-#  set_fact:
-#    mtail_installed: "{{ st.stat.exists | bool }}"
-#
-#- block:
-#  - name: determine latest version
-#    uri:
-#      url: https://api.github.com/repos/google/mtail/releases/latest
-#      return_content: true
-#      body_format: json
-#    register: _latest_version
-#    until: _latest_version.status == 200
-#    retries: 3
-#
-#  - name: set mtail_version
-#    set_fact:
-#      mtail_version: "{{ _latest_version.json['tag_name'] | regex_replace('^v', '') }}"
-#
-#- block:
-#  - name: determine installed version
-#    command: "{{ mtail_bin_path }}/mtail --version"
-#    register: _installed_version_string
-#    changed_when: false
-#
-#  - name: set mtail_local_version
-#    set_fact:
-#      mtail_local_version: "{{ _installed_version_string.stdout | regex_search(mtail_version_regex, '\\1') | first }}"
-#  when: mtail_installed
-#
-#- name: set mtail_local_version to 0
-#  set_fact:
-#    mtail_local_version: "0"
-#  when: not mtail_installed
 - name: determine if installed
  stat:
    path: "{{ mtail_bin_path }}/mtail"
--- a/roles/network/defaults/main.yml
+++ b/roles/network/defaults/main.yml
@ -6,6 +6,23 @@ network_netplan_config_path: "{{ network_netplan_etc_path }}/ansible.yaml"
 network_netplan_default_config_path: "{{ network_netplan_etc_path }}/01-netcfg.yaml"
 # network_netplan_default_config_state: absent

+network_netplan:
+  network:
+    version: 2
+    ethernets:
+      eth0:
+        dhcp4: false
+        dhcp6: false
+        accept-ra: true
+        addresses:
+          - "{{ ansible_default_ipv4.address }}/{{ (ansible_default_ipv4.network + '/' + ansible_default_ipv4.netmask) | ansible.utils.ipaddr('prefix') }}"
+          - "{{ ansible_default_ipv6.address }}/{{ ansible_default_ipv6.prefix }}"
+        routes:
+          - to: default
+            via: "{{ ansible_default_ipv4.gateway }}"
+        nameservers:
+          addresses: "{{ network_dns_nameservers }}"
+
 network_interfaces:
  - name: eth0
    inet4:
@ -15,6 +32,7 @@ network_interfaces:
      gateway: "{{ ansible_default_ipv4.gateway }}"
    inet6:
      dhcp: false
+      accept_ra: true
      address:
        - "{{ ansible_default_ipv6.address }}/{{ ansible_default_ipv6.prefix }}"
      gateway: "{{ ansible_default_ipv6.gateway }}"
--- a/roles/network/tasks/netplan.yml
+++ b/roles/network/tasks/netplan.yml
@ -5,14 +5,14 @@
    state: "{{ network_netplan_default_config_state | default('absent') }}"
    owner: root
    group: root
-    mode: 0644
+    mode: '0400'
  notify: netplan apply

 - name: Configure netplan
-  ansible.builtin.template:
+  ansible.builtin.copy:
    dest: "{{ network_netplan_config_path }}"
-    src: netplan.yaml.j2
+    content: "{{ network_netplan | to_nice_yaml }}"
    owner: root
    group: root
-    mode: '0644'
+    mode: '0400'
  notify: netplan apply
--- a/roles/network/templates/netplan.yaml.j2
+++ b/roles/network/templates/netplan.yaml.j2
@ -1,16 +1,19 @@
 ---
 network:
-  version: "{{ network_netplan_version | default(2) }}"
-  renderer: "{{ network_netplan_renderer | default("networkd") }}"
+  version: {{ network_netplan_version | default(2) }}
+  renderer: {{ network_netplan_renderer | default('networkd') }}
 {% if network_interfaces is defined and network_interfaces | length %}
  ethernets:
 {%   for iface in network_interfaces %}
    {{ iface['name'] }}:
 {%     if iface['inet4']['dhcp'] is defined %}
-      dhcp4: "{{ iface['inet4']['dhcp'] | ternary('yes', 'no') }}"
+      dhcp4: {{ iface['inet4']['dhcp'] | ternary('true', 'false') }}
 {%     endif %}
 {%     if iface['inet4']['dhcp'] is defined %}
-      dhcp6: "{{ iface['inet6']['dhcp'] | ternary('yes', 'no') }}"
+      dhcp6: {{ iface['inet6']['dhcp'] | ternary('true', 'false') }}
+{%     endif %}
+{%     if iface['inet6']['accept_ra'] is defined %}
+      accept-ra: {{ iface['inet6']['accept_ra'] | ternary('true', 'false') }}
 {%     endif %}
 {%     if iface['inet4']['address'] is defined or iface['inet6']['address'] is defined %}
      addresses:
@ -22,10 +25,10 @@ network:
 {%       endfor %}
 {%     endif %}
 {%     if iface['inet4']['gateway'] is defined %}
-      gateway4: "{{ iface['inet4']['gateway'] }}"
+      gateway4: {{ iface['inet4']['gateway'] }}
 {%     endif %}
 {%     if iface['inet6']['gateway'] is defined %}
-      gateway6: "{{ iface['inet6']['gateway'] }}"
+      gateway6: {{ iface['inet6']['gateway'] }}
 {%     endif %}
 {%     if network_dns_nameservers is defined %}
      nameservers:
--- a/roles/nftables/defaults/main.yaml
+++ b/roles/nftables/defaults/main.yaml
@ -0,0 +1,89 @@
+---
+# nftables_ufw_package_name: ufw
+# nftables_ufw_package_state: absent
+
+# nftables_package_name: nftables
+# nftables_package_state: present
+
+# nftables_service_name: nftables
+# nftables_service_state: started
+# nftables_service_enabled: true
+
+# nftables_config_path: /etc/nftables.conf
+
+nftables_builtin_defines:
+  REQUIRED_ICMPV6_TYPES:
+    - 1-4
+    - 130-136
+    - 141-143
+    - 148-149
+    - 151-153
+  TRACEROUTE_UDP_PORTS: 33434-33534
+
+nftables_builtin_sets:
+  blackhole4:
+    - type ipv4_addr
+    - flags interval
+  blackhole6:
+    - type ipv6_addr
+    - flags interval
+  tcp_input_accept:
+    - type inet_service
+    - flags interval
+    - elements = { ssh }
+  udp_input_accept:
+    - type inet_service
+    - flags interval
+
+nftables_input_builtin_rules:
+  '000 policy':
+    - type filter hook input priority filter; policy drop;
+  '010 blackhole':
+    - ip saddr @blackhole4 drop
+    - ip6 saddr @blackhole6 drop
+  '020 related established':
+    - ct state established,related accept
+    - ct state invalid drop
+  '030 loopback':
+    - iifname "lo" accept
+  '040 icmp':
+    - icmpv6 type $REQUIRED_ICMPV6_TYPES accept
+    - icmpv6 type echo-request accept
+    - icmp type echo-request accept
+  '050 tcp accept':
+    - tcp dport @tcp_input_accept accept
+  '060 udp accept':
+    - udp dport @udp_input_accept accept
+  '999 traceroute':
+    # this should be last because these ports could be allowed
+    - udp dport $TRACEROUTE_UDP_PORTS reject
+
+nftables_forward_builtin_rules:
+  '000 policy':
+    - type filter hook forward priority filter; policy drop;
+  '010 related established':
+    - ct state { established, related } accept
+
+nftables_output_builtin_rules:
+  '000 policy':
+    - type filter hook output priority filter; policy accept;
+  '010 blackhole':
+    - ip daddr @blackhole4 drop
+    - ip6 daddr @blackhole6 drop
+  '020 related established':
+    - ct state { established, related } accept
+
+nftables_defines:
+  {} 
+
+nftables_sets:
+  {}
+
+nftables_input_rules:
+  {}
+
+nftables_forward_rules:
+  {}
+
+nftables_output_rules:
+  {}
--- a/roles/nftables/handlers/main.yaml
+++ b/roles/nftables/handlers/main.yaml
@ -0,0 +1,10 @@
+---
+- name: reload nftables
+  ansible.builtin.service:
+    name: "{{ nftables_service_name | default('nftables') }}"
+    state: reloaded
+
+- name: restart nftables
+  ansible.builtin.service:
+    name: "{{ nftables_service_name | default('nftables') }}"
+    state: restarted
--- a/roles/nftables/tasks/Ubuntu.yaml
+++ b/roles/nftables/tasks/Ubuntu.yaml
@ -0,0 +1,5 @@
+---
+- name: remove ufw
+  ansible.builtin.package:
+    name: "{{ nftables_ufw_package_name | default('ufw') }}"
+    state: "{{ nftables_ufw_package_state | default('absent') }}"
--- a/roles/nftables/tasks/configure.yaml
+++ b/roles/nftables/tasks/configure.yaml
@ -0,0 +1,16 @@
+---
+- name: configure rules
+  ansible.builtin.template:
+    src: nftables.conf.j2
+    dest: "{{ nftables_config_path | default('/etc/nftables.conf') }}"
+    owner: root
+    group: root
+    mode: 0600
+  notify:
+    - restart nftables
+
+- name: manage service
+  ansible.builtin.service:
+    name: "{{ nftables_service_name | default('nftables') }}"
+    state: "{{ nftables_service_state | default('started') }}"
+    enabled: "{{ nftables_service_enabled | default(true) }}"
--- a/roles/nftables/tasks/default.yaml
+++ b/roles/nftables/tasks/default.yaml
--- a/roles/nftables/tasks/install.yaml
+++ b/roles/nftables/tasks/install.yaml
@ -0,0 +1,5 @@
+---
+- name: remove ufw
+  ansible.builtin.package:
+    name: "{{ nftables_package_name | default('nftables') }}"
+    state: "{{ nftables_package_state | default('present') }}"
--- a/roles/nftables/tasks/main.yaml
+++ b/roles/nftables/tasks/main.yaml
@ -0,0 +1,28 @@
+---
+- name: gather OS specific variables
+  ansible.builtin.include_vars: "{{ lookup('ansible.builtin.first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: run os specific tasks
+  ansible.builtin.include_tasks: "{{ lookup('ansible.builtin.first_found', params) }}"
+  vars:
+    params:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- include_tasks: install.yaml
+
+- include_tasks: configure.yaml
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Ryan Cavicchioni	b45f8cf5dd	Add role for ufw	2024-04-14 18:32:50 -05:00
Ryan Cavicchioni	7caf443b35	Add draft roles for cloudflared and tailscale	2024-04-14 18:31:59 -05:00
Ryan Cavicchioni	db1ee687a7	Add vault for monitor_servers	2024-04-14 18:31:11 -05:00
Ryan Cavicchioni	e7c9f4fa05	docker: add handlers	2024-04-14 18:30:40 -05:00
Ryan Cavicchioni	22ab3586a1	lego: add configuration	2024-04-14 18:30:16 -05:00
Ryan Cavicchioni	f4585ad0ee	promtail: add configuration	2024-04-14 18:30:05 -05:00
Ryan Cavicchioni	e3549cf829	mimir: add configuration	2024-04-14 18:30:05 -05:00
Ryan Cavicchioni	04948c36b9	loki: add configuration	2024-04-14 18:30:05 -05:00
Ryan Cavicchioni	6ee8d3372a	alertmanager: configure receiver secrets	2024-04-14 18:30:05 -05:00
Ryan Cavicchioni	00ce1a8a26	Tweak rsyslog queuing	2024-04-14 18:10:35 -05:00
Ryan Cavicchioni	78835bce49	Change DNS servers	2024-04-14 18:09:13 -05:00
Ryan Cavicchioni	20db9d5088	wireguard: Use different subnet	2024-04-14 18:09:13 -05:00
Ryan Cavicchioni	55c45c6f3d	Replace certbot with lego	2024-04-14 18:09:13 -05:00
Ryan Cavicchioni	cb60bcb5f8	nginx: refactor role	2024-04-14 17:53:26 -05:00
Ryan Cavicchioni	7ca9b6dc8c	wireguard: support 'Table' and 'PersistentKeepalive'	2024-04-14 17:52:35 -05:00
Ryan Cavicchioni	0addb1e6a0	unattended-updates: enable normal updates	2024-04-14 17:52:03 -05:00
Ryan Cavicchioni	9acc10b73f	rsyslog: use variables for paths	2024-04-14 17:51:22 -05:00
Ryan Cavicchioni	01314cb137	prometheus: enable file discovery	2024-04-14 17:50:31 -05:00
Ryan Cavicchioni	1982782284	minecraft: update minecraft server	2024-04-14 17:49:36 -05:00
Ryan Cavicchioni	05b1e8da07	loki: flesh out role	2024-04-14 17:48:46 -05:00
Ryan Cavicchioni	45ddb507ef	mtail: remove dead code	2024-04-14 17:47:55 -05:00
Ryan Cavicchioni	1cce3fc642	nftables: add more rules	2024-04-14 17:46:42 -05:00
Ryan Cavicchioni	7168a89e53	Fix typos in Promtail systemd unit	2024-04-14 17:45:59 -05:00
Ryan Cavicchioni	4e338917dc	iptables: open ports for promtail syslog	2024-04-14 17:45:16 -05:00
Ryan Cavicchioni	f79cdc1e59	Update http2 syntax	2024-04-14 17:34:54 -05:00
Ryan Cavicchioni	4a7f888994	Refactor certbot role	2024-04-14 17:29:18 -05:00
Ryan Cavicchioni	8b24c9fad9	Fix pixz package name	2024-04-14 17:28:36 -05:00
Ryan Cavicchioni	77ecf4ccbe	Use tags	2024-04-14 17:26:32 -05:00
Ryan Cavicchioni	de53d99b5e	Manager restic updates	2024-04-14 17:25:38 -05:00
Ryan Cavicchioni	907d7a9c63	Add role for snmp_exporter	2024-04-14 17:23:51 -05:00
Ryan Cavicchioni	6108475fbd	Refactor netplan	2024-04-14 17:23:27 -05:00
Ryan Cavicchioni	db8c7f4f63	Secrets	2024-04-14 17:19:01 -05:00
Ryan Cavicchioni	02c1899ee0	Remove unused host_vars	2024-04-14 17:16:43 -05:00
Ryan Cavicchioni	b02da06c97	Add roles for lego, logcli, mimir, process_exporter, smokeping_prober, and vector	2024-04-14 17:13:06 -05:00
Ryan Cavicchioni	ce692e4560	Add nftables role	2022-09-04 08:59:28 -05:00
Ryan Cavicchioni	42ba49c865	common: refactor	2022-09-01 17:12:52 -05:00
Ryan Cavicchioni	4b581b8a78	restic: remove tidy job	2022-09-01 16:42:00 -05:00
Ryan Cavicchioni	132b6d800a	Remove Python 2 packages	2022-09-01 16:41:35 -05:00
Ryan Cavicchioni	2483542b98	prometheus: scrape Grafana stats	2022-09-01 16:40:12 -05:00
Ryan Cavicchioni	dae13299e0	Remove DNS zones	2022-09-01 16:39:51 -05:00
Ryan Cavicchioni	36a2d3542c	Remove name server roles	2022-09-01 16:39:28 -05:00
Ryan Cavicchioni	3fc613fe2b	grafana: add default.yaml	2022-09-01 16:37:15 -05:00
Ryan Cavicchioni	b685c1027e	Add test drone.yml Some checks failed continuous-integration/drone/push Build is failing Details continuous-integration/drone Build is failing Details	2022-09-01 09:16:08 -05:00
Ryan Cavicchioni	98b34e6c5c	grafana: add new Prometheus based system dashboard	2022-08-31 21:09:23 -05:00