add node_exporter role

roles/node_exporter/defaults/main.yaml

@@ -0,0 +1,51 @@
+---
+node_exporter_go_arch_map:
+  i386: '386'
+  x86_64: 'amd64'
+
+node_exporter_go_arch: "{{ node_exporter_go_arch_map[ansible_architecture] | default('amd64') }}"
+
+node_exporter_service_name: node_exporter.service
+node_exporter_service_enabled: true
+node_exporter_service_state: started
+
+node_exporter_version_regex: ^node_exporter, version ([\d.]+)
+
+node_exporter_release_file: "node_exporter-{{ node_exporter_version }}.{{ ansible_system | lower }}-{{ node_exporter_go_arch }}.tar.gz"
+node_exporter_release_url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/{{ node_exporter_release_file }}"
+node_exporter_checksum_url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/sha256sums.txt"
+node_exporter_download_path: "/tmp/{{ node_exporter_release_file }}"
+node_exporter_unarchive_dest_path: /tmp
+node_exporter_extracted_path: "{{ node_exporter_download_path | replace('.tar.gz', '') }}"
+
+node_exporter_user: node_exporter
+node_exporter_user_state: present
+node_exporter_user_shell: /usr/sbin/nologin
+
+node_exporter_group: node_exporter
+node_exporter_group_state: "{{ node_exporter_user_state | default('present') }}"
+
+node_exporter_var_path: /var/lib/node_exporter
+node_exporter_var_owner: "{{ node_exporter_user }}"
+node_exporter_var_group: "{{ node_exporter_group }}"
+node_exporter_var_mode: "0755"
+
+node_exporter_spool_path: /var/spool/node_exporter
+node_exporter_spool_owner: "{{ node_exporter_user }}"
+node_exporter_spool_group: "{{ node_exporter_group }}"
+node_exporter_spool_mode: "0755"
+
+node_exporter_bin_path: /usr/local/bin
+
+node_exporter_collectors_enabled:
+  - textfile:
+      directory: "{{ node_exporter_spool_path }}/textfile_collector"
+  - processes
+  - tcpstat
+  - ntp
+  - supervisord:
+      url: unix:///var/run/supervisor.sock
+  - systemd:
+      enable-task-metrics:
+      enable-restarts-metrics:
+      enable-start-time-metrics:

roles/node_exporter/files/apt-exporter.pl

@@ -0,0 +1,36 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+my $cmd = "apt-get --just-print dist-upgrade";
+my %metrics;
+
+open(my $fh, '-|', $cmd) or die $!;
+while(my $line = <$fh>) {
+    if ($line =~ /Inst \S+ \S+ \(\S+ (.+) \[(\S+)\]\)/) {
+        my $k = sprintf("apt_upgrades_pending{origin=\"%s\", arch=\"%s\"}", $1, $2); 
+        if (!exists $metrics{$k}) {
+            $metrics{$k} = 1;
+        } else {
+            $metrics{$k}++;
+        }
+    }
+}
+
+if (%metrics) {
+    # print apt metrics
+    while(my($k, $v) = each %metrics) {
+        printf("%s %d\n", $k, $v)
+    }
+}
+else {
+    print("apt_upgrades_pending{origin=\"\",arch=\"\"} 0\n");
+}
+
+# print reboot required metric
+if (-e "/var/run/reboot-required") {
+    print("node_reboot_required 1\n")
+}
+else {
+    print("node_reboot_required 0\n")
+}

roles/node_exporter/files/promcat.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+function usage { printf "Usage: %s FILE\n" "$(basename "$0")" >&2; exit 1; }
+
+while getopts "h" opt; do
+    case "${opt}" in
+        *)
+            usage
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+FILE="$1"
+
+if [ -z "${FILE}" ]; then
+    usage
+    exit 1
+fi
+
+if command -v sponge > /dev/null; then
+    ( echo "# promcat (sponge)" ; cat /dev/stdin ) | sponge "${FILE}"
+else
+    TEMP=$(mktemp --suffix .prom)
+
+    function finish {
+        if [ -f "${TEMP}" ]; then
+            rm -f "${TEMP}"
+        fi
+    }
+    trap finish EXIT
+
+    echo "# promcat (mktemp, mv)" > "${TEMP}"
+    cat /dev/stdin >> "${TEMP}"
+
+    if [ ! -s "${TEMP}" ] || grep -q '^[[:space:]]*$' "${TEMP}" ; then
+        printf "%s is empty\n" "${TEMP}" >&2
+        exit 1
+    else
+        mv "${TEMP}" "${FILE}"
+    fi
+fi

roles/node_exporter/handlers/main.yaml

@@ -0,0 +1,6 @@
+---
+- name: restart node_exporter
+  systemd:
+    name: node_exporter.service
+    daemon_reload: true
+    state: restarted

roles/node_exporter/tasks/Debian.yaml

@@ -0,0 +1,25 @@
+---
+- name: reboot required cron
+  cron:
+    name: "prometheus reboot required"
+    minute: "0"
+    job: |
+      [ ! -f /var/run/reboot-required ] ; printf "node_reboot_required \%s\n" "$?" | {{ node_exporter_bin_path }}/promcat {{ node_exporter_spool_path }}/textfile_collector/node_reboot_required.prom
+    user: root
+    state: absent
+
+- name: copy apt-exporter
+  copy:
+    src: apt-exporter.pl
+    dest: "{{ node_exporter_bin_path }}/apt-exporter"
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: apt metrics cron
+  cron:
+    name: prometheus apt exporter
+    minute: "0"
+    job: |
+      {{ node_exporter_bin_path }}/apt-exporter | {{ node_exporter_bin_path }}/promcat {{ node_exporter_spool_path }}/textfile_collector/apt.prom
+    user: root

roles/node_exporter/tasks/configure.yaml

@@ -0,0 +1,87 @@
+---
+- name: create group
+  group:
+    name: "{{ node_exporter_group }}"
+    system: true
+    state: "{{ node_exporter_group_state | default('present') }}"
+
+- name: create user
+  user:
+    name: "{{ node_exporter_user }}"
+    system: true
+    shell: "{{ node_exporter_user_shell }}"
+    group: "{{ node_exporter_group }}"
+    createhome: false
+    home: "{{ node_exporter_var_path }}"
+    state: "{{ node_exporter_user_state | default('present') }}"
+
+      #- name: create etc path
+      #  file:
+      #    path: "{{ node_exporter_etc_path }}"
+      #    state: directory
+      #    owner: "{{ node_exporter_etc_owner }}"
+      #    group: "{{ node_exporter_etc_group }}"
+      #    mode: "{{ node_exporter_etc_mode }}"
+
+- name: create var path
+  file:
+    path: "{{ node_exporter_var_path }}"
+    state: directory
+    owner: "{{ node_exporter_var_owner }}"
+    group: "{{ node_exporter_var_group }}"
+    mode: "{{ node_exporter_var_mode }}"
+
+- name: create spool path
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ node_exporter_spool_owner }}"
+    group: "{{ node_exporter_spool_group }}"
+    mode: "{{ node_exporter_spool_mode }}"
+  loop:
+    - "{{ node_exporter_spool_path }}"
+    - "{{ node_exporter_spool_path }}/textfile_collector"
+
+- name: configure systemd template
+  template:
+    src: node_exporter.service.j2
+    dest: /etc/systemd/system/node_exporter.service
+    owner: root
+    group: root
+    mode: 0444
+  notify: restart node_exporter
+
+- name: manage service
+  service:
+    name: "{{ node_exporter_service_name }}"
+    enabled: "{{ node_exporter_service_enabled }}"
+    state: "{{ node_exporter_service_state }}"
+
+- name: install helper scripts
+  copy:
+    src: "{{ item }}.sh"
+    dest: "{{ node_exporter_bin_path }}/{{ item }}"
+    owner: root
+    group: root
+    mode: '0755'
+  loop:
+    - promcat
+
+- name: node_exporter directory size cron
+  cron:
+    name: node_exporter directory size
+    minute: "*/5"
+    job: >
+      du -sb {{ node_exporter_du_directories | join(' ') }} |
+      sed -ne 's/^\([0-9]\+\)\t\(.*\)$/node_directory_size_bytes{directory="\2"} \1/p' |
+      {{ node_exporter_bin_path }}/promcat {{ node_exporter_spool_path }}/textfile_collector/directory_size.prom
+    user: root
+    state: "{{ (node_exporter_du_directories | length > 0) | ternary('present', 'absent') }}"
+
+- name: configure roles
+  template:
+    src: roles.prom.j2
+    dest: "{{ node_exporter_spool_path }}/textfile_collector/roles.prom"
+    owner: root
+    group: root
+    mode: "0444"

roles/node_exporter/tasks/install.yaml

@@ -0,0 +1,32 @@
+---
+- block:
+  - name: download tar
+    get_url:
+      url: "{{ node_exporter_release_url }}"
+      dest: "{{ node_exporter_download_path }}"
+      checksum: "{{ node_exporter_checksum }}"
+    register: dl
+    until: dl is success
+    retries: 5
+    delay: 10
+
+  - name: extract tar
+    unarchive:
+      src: "{{ node_exporter_download_path }}"
+      dest: "{{ node_exporter_unarchive_dest_path }}"
+      creates: "{{ node_exporter_extracted_path }}/node_exporter"
+      remote_src: true
+
+  - name: install binaries
+    copy:
+      src: "{{ node_exporter_extracted_path }}/{{ item }}"
+      dest: "{{ node_exporter_bin_path }}/{{ item }}"
+      owner: root
+      group: root
+      mode: 0755
+      remote_src: true
+    loop:
+      - node_exporter
+    notify: restart node_exporter
+  when: node_exporter_version != node_exporter_local_version
+

roles/node_exporter/tasks/main.yaml

@@ -0,0 +1,32 @@
+---
+- name: gather os specific variables
+  include_vars: "{{ lookup('first_found', possible_files) }}"
+  vars:
+    possible_files:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - vars
+
+- name: include os specific tasks
+  include_tasks: "{{ lookup('first_found', possible_files) }}"
+  vars:
+    possible_files:
+      files:
+        - "{{ ansible_distribution }}-{{ ansible_distribution_version }}.yaml"
+        - "{{ ansible_distribution }}.yaml"
+        - "{{ ansible_os_family }}.yaml"
+        - "default.yaml"
+      paths:
+        - tasks
+
+- include: pre.yaml
+
+- include: install.yaml
+  when:
+    - node_exporter_version is version(node_exporter_local_version, '>')
+
+- include: configure.yaml

roles/node_exporter/tasks/pre.yaml

@@ -0,0 +1,50 @@
+---
+- name: determine if installed
+  stat:
+    path: "{{ node_exporter_bin_path }}/node_exporter"
+  register: st
+
+- name: set node_exporter_installed
+  set_fact:
+    node_exporter_installed: "{{ st.stat.exists | bool }}"
+
+- block:
+  - name: determine latest version
+    uri:
+      url: https://api.github.com/repos/prometheus/node_exporter/releases/latest
+      return_content: true
+      body_format: json
+    register: _latest_version
+    until: _latest_version.status == 200
+    retries: 3
+
+  - name: set node_exporter_version
+    set_fact:
+      node_exporter_version: "{{ _latest_version.json['tag_name'] | regex_replace('^v', '') }}"
+
+- block:
+  - name: determine installed version
+    command: "{{ node_exporter_bin_path }}/node_exporter --version"
+    register: _installed_version_string
+    changed_when: false
+
+  - name: set node_exporter_local_version
+    set_fact:
+      node_exporter_local_version: "{{ _installed_version_string.stdout | regex_search(node_exporter_version_regex, '\\1') | first }}"
+  when: node_exporter_installed
+
+- name: set node_exporter_local_version to 0
+  set_fact:
+    node_exporter_local_version: "0"
+  when: not node_exporter_installed
+
+- block:
+  - name: get checksums
+    set_fact:
+      _checksums: "{{ lookup('url', node_exporter_checksum_url, wantlist=True) }}"
+
+  - name: set node_exporter_checksum
+    set_fact:
+      node_exporter_checksum: "sha256:{{ item.split(' ') | first }}"
+    loop: "{{ _checksums }}"
+    when: "node_exporter_release_file in item"

roles/node_exporter/templates/node_exporter.service.j2

@@ -0,0 +1,32 @@
+[Unit]
+Description=Node Exporter
+After=network-online.target
+
+[Service]
+Type=simple
+User={{ node_exporter_user }}
+Group={{ node_exporter_group }}
+ExecStart={{ node_exporter_bin_path }}/node_exporter \
+{% if node_exporter_collectors_enabled %}
+{%     for collector in node_exporter_collectors_enabled %}
+{%         if collector is mapping %}
+{%             set name, opts = (collector.items() | list)[0] %}
+    --collector.{{ name }} \
+{%             for k, v in opts.items() %}
+{%                 if v is none %}
+    --collector.{{ name }}.{{ k }} \
+{%                 elif v is string %}
+    --collector.{{ name }}.{{ k }}={{ v }} \
+{%                 endif %}
+{%             endfor %}
+{%         else %}
+    --collector.{{ collector }} \
+{%         endif %}
+{%     endfor %}
+{% endif %}
+
+Restart=always
+RestartSec=1
+
+[Install]
+WantedBy=multi-user.target

roles/node_exporter/templates/roles.prom.j2

@@ -0,0 +1,5 @@
+{% if node_exporter_machine_roles is defined %}
+{% for role in node_exporter_machine_roles %}
+machine_role{role="{{ role }}"} 1
+{% endfor %}
+{% endif %}