add restic wrapper scripts

roles/restic/defaults/main.yaml

@@ -11,5 +11,16 @@ restic_bin_path: /usr/local/bin
 restic_etc_path: /etc/restic
 restic_path: "{{ restic_bin_path }}/restic"
 
+restic_global_exclude:
+  - /dev
+  - /media
+  - /mnt
+  - /proc
+  - /run
+  - /sys
+  - /tmp
+  - /var/tmp
+  - /var/lib/lxcfs/cgroup
+
 restic_repos: []
 restic_jobs: []

roles/restic/tasks/job.yaml

@@ -7,6 +7,14 @@
     mode: 0755
     state: directory
 
+- name: create repo environment helper
+  template:
+    src: job-env.sh.j2
+    dest: "{{ restic_etc_path }}/jobs/{{ item.name }}/env.sh"
+    owner: root
+    group: root
+    mode: 0400
+
 - name: create job exclude file
   template:
     src: exclude.txt.j2
@@ -19,10 +27,10 @@
   cron:
     name: "restic {{ item.name }} job"
     hour: "{{ item.cron.hour | default(omit) }}"
-    minute: "{{ item.cron.minute | default(omit) }}"
+    minute: "{{ item.cron.minute | default(60 | random(seed=inventory_hostname)) }}"
     day: "{{ item.cron.day | default(omit) }}"
     month: "{{ item.cron.month | default(omit) }}"
     weekday: "{{ item.cron.weekday | default(omit) }}"
     user: "{{ item.cron.user | default('root') }}"
     state: "{{ item.cron.state | default('present') }}"
-    job: ". {{ restic_etc_path }}/repos/{{ item.repo }}/env.sh && restic backup -q --exclude-file {{ restic_etc_path }}/jobs/{{ item.name }}/exclude.txt {{ item.paths | join(' ') }}"
+    job: "{{ restic_bin_path }}/restic-job {{ item.name }}"

roles/restic/tasks/main.yaml

@@ -65,6 +65,22 @@
     - "{{ restic_etc_path }}/repos"
     - "{{ restic_etc_path }}/jobs"
 
+- name: create restic job wrapper script
+  template:
+    src: restic-job.sh.j2
+    dest: "{{ restic_bin_path }}/restic-job"
+    owner: root
+    group: root
+    mode: 0755
+
+- name: create restic tidy wrapper
+  template:
+    src: restic-tidy.sh.j2
+    dest: "{{ restic_bin_path }}/restic-tidy"
+    owner: root
+    group: root
+    mode: 0755
+
 - name: manage repos
   include: repo.yaml
   loop: "{{ restic_repos | default([]) }}"

roles/restic/tasks/repo.yaml

@@ -1,7 +1,7 @@
 ---
 - name: get repo status
   shell:
-    cmd: restic -q -r {{ item.repo }} snapshots 2> /dev/null 
+    cmd: restic -q -r {{ item.repo }} --no-lock snapshots 2> /dev/null 
   ignore_errors: yes
   environment: "{{ item.environment | default({}) }}"
   register: restic_init
@@ -30,8 +30,17 @@
 
 - name: create repo environment helper
   template:
-    src: env.sh.j2
+    src: repo-env.sh.j2
     dest: "{{ restic_etc_path }}/repos/{{ item.name }}/env.sh"
     owner: root
     group: root
     mode: 0400
+
+- name: create cron
+  cron:
+    name: "restic {{ item.name }} tidy"
+    hour: "0"
+    minute: "{{ 60 | random(seed=inventory_hostname) }}"
+    user: root
+    state: present
+    job: "{{ restic_bin_path }}/restic-tidy {{ item.name }}"

roles/restic/templates/env.sh.j2

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-export RESTIC_REPOSITORY="{{ item.repo }}"
-{% for k, v in item.environment.items() %}
-export {{ k | upper }}="{{ v }}"
-{% endfor %}

roles/restic/templates/exclude.txt.j2

@@ -1,3 +1,15 @@
+# {{ ansible_managed }}
+
+{% if restic_global_exclude is defined %}
+# global exclusions
+{% for exclude in restic_global_exclude | default([]) %}
+{{ exclude }}
+{% endfor %}
+{% endif %}
+
+{% if item.exclude is defined %}
+# job exclusions
 {% for exclude in item.exclude %}
 {{ exclude }}
 {% endfor %}
+{%- endif -%}

roles/restic/templates/job-env.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# {{ ansible_managed }}
+
+REPO="{{ item.repo }}"
+PATHS="{{ item.paths | join(' ') }}"

roles/restic/templates/repo-env.sh.j2

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# {{ ansible_managed }}
+
+export RESTIC_REPOSITORY="{{ item.repo }}"
+{% if "environment" in item %}
+{% for k, v in item.environment.items() %}
+export {{ k | upper }}="{{ v }}"
+{% endfor %}
+{% endif %}
+
+{% if "forget" in item %}
+{% for k, v in item.forget.items() %}
+{{ k | upper }}="{{ v }}"
+{% endfor %}
+{% endif %}

roles/restic/templates/restic-job.sh.j2

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# {{ ansible_managed }}
+
+error_exit() {
+    printf "%s\n" "$1"
+    exit 1
+}
+
+NICE="ionice -c2 nice -n19"
+JOB=$1
+
+if [ -z "$JOB" ]; then
+    error_exit "job is missing"
+fi
+
+RESTIC_ETC_PATH="{{ restic_etc_path }}"
+JOB_PATH="${RESTIC_ETC_PATH}/jobs/${JOB}"
+JOB_ENV="${JOB_PATH}/env.sh"
+
+if [ ! -r "$JOB_ENV" ]; then
+    error_exit "${JOB_ENV} does not exist"
+fi
+
+. "$JOB_ENV"
+
+if [ -z "${REPO+x}" ]; then
+    error_exit "\$REPO is not set"
+fi
+
+REPO_PATH="${RESTIC_ETC_PATH}/repos/${REPO}"
+REPO_ENV="${REPO_PATH}/env.sh"
+
+if [ ! -r "$REPO_ENV" ]; then
+    error_exit "${REPO_ENV} does not exist"
+fi
+
+. "$REPO_ENV"
+
+EXCLUDE_PATH="${JOB_PATH}/exclude.txt"
+
+if [ -z "${PATHS+x}" ]; then
+    error_exit "\$PATHS is not set"
+fi
+
+if [ -r "$EXCLUDE_PATH" ]; then
+    $NICE {{ restic_path }} backup -q --exclude-file="${EXCLUDE_PATH}" "${PATHS}"
+else
+    $NICE {{ restic_path }} backup -q "${PATHS}"
+fi

roles/restic/templates/restic-tidy.sh.j2

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# {{ ansible_managed }}
+
+error_exit() {
+    printf "%s\n" "$1"
+    exit 1
+}
+
+MAX_ATTEMPTS=60
+REPO=$1
+
+if [ -z "$REPO" ]; then
+    error_exit "repo is missing"
+fi
+
+RESTIC_ETC_PATH="{{ restic_etc_path }}"
+REPO_PATH="${RESTIC_ETC_PATH}/repos/${REPO}"
+REPO_ENV="${REPO_PATH}/env.sh"
+
+if [ ! -r "$REPO_ENV" ]; then
+    error_exit "${REPO_ENV} does not exist"
+fi
+
+. "$REPO_ENV"
+
+KEEP_DAILY=${KEEP_DAILY:-7}
+KEEP_WEEKLY=${KEEP_WEEKLY:-5}
+KEEP_MONTHLY=${KEEP_MONTHLY:-12}
+KEEP_YEARLY=${KEEP_YEARLY:-10}
+
+counter=0
+sleep=1
+rc=1
+
+until [ $counter -eq $MAX_ATTEMPTS ] || [ $rc -eq 0 ]; do
+  {{ restic_path }} forget \
+      --quiet \
+      --host $(hostname -f) \
+      --keep-daily "$KEEP_DAILY" \
+      --keep-weekly "$KEEP_WEEKLY" \
+      --keep-monthly "$KEEP_MONTHLY" \
+      --keep-yearly "$KEEP_YEARLY" \
+      --prune
+
+  rc=$?
+
+  if [ $rc -ne 0 ]; then
+    sleep=$((counter * 5))
+    printf "sleeping for %d seconds (%d)\n" $sleep $counter
+    sleep $sleep
+  fi
+
+  let counter+=1
+done
+
+if [ $rc -ne 0 ] && [ $counter -eq $MAX_ATTEMPTS ]; then
+  printf "tidy timed out, exiting\n"
+fi