add mtail role

roles/mtail/defaults/main.yaml

@@ -0,0 +1,50 @@
+---
+mtail_go_arch_map:
+  i386: '386'
+  x86_64: 'amd64'
+
+mtail_go_arch: "{{ mtail_go_arch_map[ansible_architecture] | default('amd64') }}"
+
+mtail_service_name: mtail.service
+mtail_service_state: started
+mtail_service_enabled: yes
+
+mtail_version_regex: ^mtail version (\S+)
+
+mtail_github_project_url: https://github.com/google/mtail
+mtail_release_file: "mtail_{{ mtail_version }}_{{ ansible_system | capitalize }}_{{ ansible_architecture }}.tar.gz"
+mtail_release_url: "{{ mtail_github_project_url }}/releases/download/v{{ mtail_version }}/{{ mtail_release_file }}"
+mtail_download_path: "/tmp/{{ mtail_release_file }}"
+mtail_checksum_url: "{{ mtail_github_project_url }}/releases/download/v{{ mtail_version }}/checksums.txt"
+mtail_extracted_path: "/tmp"
+mtail_unarchive_dest_path: "/tmp"
+
+mtail_user: mtail
+mtail_user_state: present
+mtail_user_shell: /usr/sbin/nologin
+mtail_append_groups:
+  - adm
+
+mtail_group: mtail
+mtail_group_state: "{{ mtail_user_state | default('present') }}"
+
+mtail_etc_path: /etc/mtail
+mtail_etc_owner: root
+mtail_etc_group: root
+mtail_etc_mode: "0755"
+
+mtail_var_path: /var/lib/mtail
+mtail_var_owner: "{{ mtail_user }}"
+mtail_var_group: "{{ mtail_group }}"
+mtail_var_mode: "0755"
+
+mtail_var_log_path: /var/log/mtail
+mtail_var_log_owner: "{{ mtail_user }}"
+mtail_var_log_group: "{{ mtail_group }}"
+mtail_var_log_mode: "0755"
+
+mtail_bin_path: /usr/local/bin
+
+mtail_arg_logs:
+  - "/var/log/syslog/{{ inventory_hostname_short }}/*/*/*.log"   
+  - /var/log/nginx/*.access.log

roles/mtail/files/rules/nginx.mtail

@@ -0,0 +1,29 @@
+getfilename() !~ /nginx\/.*\.log$/ {
+    stop
+}
+
+counter nginx_http_requests_total by vhost, method, code
+counter nginx_http_response_size_bytes_total by vhost, method, code
+
+histogram nginx_http_response_time_seconds buckets 0.0, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 50.0 by vhost, method, code
+
+/^/ +
+/(?P<vhost>[0-9A-Za-z\.\-:]+) / +
+/(?P<remote_addr>\S+) / +
+/- / + 
+/(?P<remote_user>\S+) / +
+/\[(?P<time_local>\d{2}\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} (\+|-)\d{4})\] / +
+/"(?P<request_method>[A-Z]+) (?P<request_uri>\S+) (?P<http_version>HTTP\/[0-9\.]+)" / +
+/(?P<status>\d{3}) / +
+/(?P<bytes_sent>\d+) / +
+/(?P<request_time>\d+\.\d+) / +
+/"(?P<http_referer>\S+)" / +
+/"(?P<http_user_agent>[[:print:]]+)" / +
+/"(?P<http_x_forwarded_for>\S+)"/ +
+/$/ {
+    nginx_http_requests_total[$vhost][$request_method][$status]++
+
+    nginx_http_response_size_bytes_total[$vhost][$request_method][$status] += $bytes_sent
+
+    nginx_http_response_time_seconds[$vhost][$request_method][$status] = $request_time
+}

roles/mtail/files/rules/syslog.mtail

@@ -0,0 +1,32 @@
+getfilename() !~ /^\/var\/log\/syslog\// {
+    stop
+}
+
+def syslog {
+    /(?P<date>(?P<legacy_date>\w+\s+\d+\s+\d+:\d+:\d+)|(?P<rfc3339_date>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+[+-]\d{2}:\d{2}))/ +
+        /\s+(?:\w+@)?(?P<hostname>[\w\.-]+)\s+(?P<application>[\w\.-]+)(?:\[(?P<pid>\d+)\])?:\s+(?P<message>.*)/ {
+        # If the legacy_date regexp matched, try this format.
+        len($legacy_date) > 0 {
+            strptime($legacy_date, "Jan _2 15:04:05")
+        }
+        # If the RFC3339 style matched, parse it this way.
+        len($rfc3339_date) > 0 {
+            strptime($rfc3339_date, "2006-01-02T15:04:05-07:00")
+        }
+        # Call into the decorated block
+        next
+    }
+}
+
+counter syslog_loglines_total by application
+counter ssh_invalid_user
+
+@syslog {
+    syslog_loglines_total[$application]++
+    $application == "sshd" {
+        $message =~ /^Invalid user/ {
+            ssh_invalid_user++
+        }
+    }
+}
+

roles/mtail/handlers/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: restart mtail
+  systemd:
+    name: mtail.service
+    daemon_reload: true
+    state: restarted

roles/mtail/tasks/configure.yaml

@@ -0,0 +1,67 @@
+---
+- name: create group
+  group:
+    name: "{{ mtail_group }}"
+    system: true
+    state: "{{ mtail_group_state | default('present') }}"
+
+- name: create user
+  user:
+    name: "{{ mtail_user }}"
+    system: true
+    shell: "{{ mtail_user_shell }}"
+    group: "{{ mtail_group }}"
+    groups: "{{ [mtail_group] + (mtail_append_groups | default([])) }}"
+    append: true
+    createhome: false
+    home: "{{ mtail_var_path }}"
+    state: "{{ mtail_user_state | default('present') }}"
+
+- name: create etc path
+  file:
+    path: "{{ mtail_etc_path }}"
+    state: directory
+    owner: "{{ mtail_etc_owner }}"
+    group: "{{ mtail_etc_group }}"
+    mode: "{{ mtail_etc_mode }}"
+
+- name: create var path
+  file:
+    path: "{{ mtail_var_path }}"
+    state: directory
+    owner: "{{ mtail_var_owner }}"
+    group: "{{ mtail_var_group }}"
+    mode: "{{ mtail_var_mode }}"
+
+- name: create var_log path
+  file:
+    path: "{{ mtail_var_log_path }}"
+    state: directory
+    owner: "{{ mtail_var_log_owner }}"
+    group: "{{ mtail_var_log_group }}"
+    mode: "{{ mtail_var_log_mode }}"
+
+- name: configure rules
+  copy:
+    src: "{{ item }}"
+    dest: "{{ mtail_etc_path }}/{{ item | basename }}"
+    owner: root
+    group: root
+    mode: "0755"
+  loop: "{{ lookup('fileglob', 'rules/*.mtail', wantlist=True) }}"
+  notify: restart mtail
+
+- name: configure systemd template
+  template:
+    src: mtail.service.j2
+    dest: /etc/systemd/system/mtail.service
+    owner: root
+    group: root
+    mode: 0444
+  notify: restart mtail
+
+- name: manage service
+  service:
+    name: "{{ mtail_service_name }}"
+    enabled: "{{ mtail_service_enabled }}"
+    state: "{{ mtail_service_state }}"

roles/mtail/tasks/install.yaml

@@ -0,0 +1,52 @@
+---
+#- block:
+#  - name: download tar
+#    get_url:
+#      url: "{{ mtail_release_url }}"
+#      dest: "{{ mtail_download_path }}"
+#    register: dl
+#    until: dl is success
+#    retries: 5
+#    delay: 10
+#
+#  - name: install binaries
+#    copy:
+#      src: "{{ mtail_download_path }}"
+#      dest: "{{ mtail_bin_path }}/mtail"
+#      owner: root
+#      group: root
+#      mode: 0755
+#      remote_src: true
+#    notify: restart mtail
+#  when: mtail_version != mtail_local_version
+#
+- block:
+  - name: download tar
+    get_url:
+      url: "{{ mtail_release_url }}"
+      dest: "{{ mtail_download_path }}"
+      checksum: "{{ mtail_checksum }}"
+    register: dl
+    until: dl is success
+    retries: 5
+    delay: 10
+
+  - name: extract tar
+    unarchive:
+      src: "{{ mtail_download_path }}"
+      dest: "{{ mtail_unarchive_dest_path }}"
+      creates: "{{ mtail_extracted_path }}/mtail"
+      remote_src: true
+
+  - name: install binaries
+    copy:
+      src: "{{ mtail_extracted_path }}/{{ item }}"
+      dest: "{{ mtail_bin_path }}/{{ item }}"
+      owner: root
+      group: root
+      mode: 0755
+      remote_src: true
+    loop:
+      - mtail
+    notify: restart mtail
+  when: mtail_version != mtail_local_version

roles/mtail/tasks/main.yaml

@@ -0,0 +1,30 @@
+---
+- name: gather os specific variables
+  include_vars: "{{ lookup('first_found', possible_files) }}"
+  vars:
+    possible_files:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: include os specific tasks
+  include_tasks: "{{ lookup('first_found', possible_files) }}"
+  vars:
+    possible_files:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- include: pre.yaml
+
+- include: install.yaml
+
+- include: configure.yaml

roles/mtail/tasks/pre.yaml

@@ -0,0 +1,88 @@
+---
+#- name: determine if installed
+#  stat:
+#    path: "{{ mtail_bin_path }}/mtail"
+#  register: st
+#
+#- name: set mtail_installed
+#  set_fact:
+#    mtail_installed: "{{ st.stat.exists | bool }}"
+#
+#- block:
+#  - name: determine latest version
+#    uri:
+#      url: https://api.github.com/repos/google/mtail/releases/latest
+#      return_content: true
+#      body_format: json
+#    register: _latest_version
+#    until: _latest_version.status == 200
+#    retries: 3
+#
+#  - name: set mtail_version
+#    set_fact:
+#      mtail_version: "{{ _latest_version.json['tag_name'] | regex_replace('^v', '') }}"
+#
+#- block:
+#  - name: determine installed version
+#    command: "{{ mtail_bin_path }}/mtail --version"
+#    register: _installed_version_string
+#    changed_when: false
+#
+#  - name: set mtail_local_version
+#    set_fact:
+#      mtail_local_version: "{{ _installed_version_string.stdout | regex_search(mtail_version_regex, '\\1') | first }}"
+#  when: mtail_installed
+#
+#- name: set mtail_local_version to 0
+#  set_fact:
+#    mtail_local_version: "0"
+#  when: not mtail_installed
+- name: determine if installed
+  stat:
+    path: "{{ mtail_bin_path }}/mtail"
+  register: st
+
+- name: set mtail_installed
+  set_fact:
+    mtail_installed: "{{ st.stat.exists | bool }}"
+
+- block:
+  - name: determine latest version
+    uri:
+      url: https://api.github.com/repos/google/mtail/releases/latest
+      return_content: true
+      body_format: json
+    register: _latest_version
+    until: _latest_version.status == 200
+    retries: 3
+
+  - name: set mtail_version
+    set_fact:
+      mtail_version: "{{ _latest_version.json['tag_name'] | regex_replace('^v', '') }}"
+
+- block:
+  - name: determine installed version
+    command: "{{ mtail_bin_path }}/mtail --version"
+    register: _installed_version_string
+    changed_when: false
+
+  - name: set mtail_local_version
+    set_fact:
+      mtail_local_version: "{{ _installed_version_string.stdout | regex_search(mtail_version_regex, '\\1') | first }}"
+  when: mtail_installed
+
+- name: set mtail_local_version to 0
+  set_fact:
+    mtail_local_version: "0"
+  when: not mtail_installed
+
+- block:
+  - name: get checksums
+    set_fact:
+      _checksums: "{{ lookup('url', mtail_checksum_url, wantlist=True) }}"
+
+  - name: set mtail_checksum
+    set_fact:
+      mtail_checksum: "sha256:{{ item.split(' ') | first }}"
+    loop: "{{ _checksums }}"
+    when: "mtail_release_file in item"

roles/mtail/templates/mtail.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=mtail
+
+[Service]
+User={{ mtail_user }}
+ExecStart={{ mtail_bin_path }}/mtail \
+  --progs {{ mtail_etc_path }} \
+  --log_dir={{ mtail_var_log_path }} \
+{% if mtail_arg_logs %}
+{% for path in  mtail_arg_logs %}
+  --logs {{ path }} \
+{% endfor %}
+{% endif %}
+
+[Install]
+WantedBy=multi-user.target