diff --git a/roles/restic/defaults/main.yaml b/roles/restic/defaults/main.yaml index e251128..2091ee4 100644 --- a/roles/restic/defaults/main.yaml +++ b/roles/restic/defaults/main.yaml @@ -11,5 +11,16 @@ restic_bin_path: /usr/local/bin restic_etc_path: /etc/restic restic_path: "{{ restic_bin_path }}/restic" +restic_global_exclude: + - /dev + - /media + - /mnt + - /proc + - /run + - /sys + - /tmp + - /var/tmp + - /var/lib/lxcfs/cgroup + restic_repos: [] restic_jobs: [] diff --git a/roles/restic/tasks/job.yaml b/roles/restic/tasks/job.yaml index 5f076cd..e0e308d 100644 --- a/roles/restic/tasks/job.yaml +++ b/roles/restic/tasks/job.yaml @@ -7,6 +7,14 @@ mode: 0755 state: directory +- name: create repo environment helper + template: + src: job-env.sh.j2 + dest: "{{ restic_etc_path }}/jobs/{{ item.name }}/env.sh" + owner: root + group: root + mode: 0400 + - name: create job exclude file template: src: exclude.txt.j2 @@ -19,10 +27,10 @@ cron: name: "restic {{ item.name }} job" hour: "{{ item.cron.hour | default(omit) }}" - minute: "{{ item.cron.minute | default(omit) }}" + minute: "{{ item.cron.minute | default(60 | random(seed=inventory_hostname)) }}" day: "{{ item.cron.day | default(omit) }}" month: "{{ item.cron.month | default(omit) }}" weekday: "{{ item.cron.weekday | default(omit) }}" user: "{{ item.cron.user | default('root') }}" state: "{{ item.cron.state | default('present') }}" - job: ". {{ restic_etc_path }}/repos/{{ item.repo }}/env.sh && restic backup -q --exclude-file {{ restic_etc_path }}/jobs/{{ item.name }}/exclude.txt {{ item.paths | join(' ') }}" + job: "{{ restic_bin_path }}/restic-job {{ item.name }}" diff --git a/roles/restic/tasks/main.yaml b/roles/restic/tasks/main.yaml index a6c5430..7df4e3e 100644 --- a/roles/restic/tasks/main.yaml +++ b/roles/restic/tasks/main.yaml @@ -65,6 +65,22 @@ - "{{ restic_etc_path }}/repos" - "{{ restic_etc_path }}/jobs" +- name: create restic job wrapper script + template: + src: restic-job.sh.j2 + dest: "{{ restic_bin_path }}/restic-job" + owner: root + group: root + mode: 0755 + +- name: create restic tidy wrapper + template: + src: restic-tidy.sh.j2 + dest: "{{ restic_bin_path }}/restic-tidy" + owner: root + group: root + mode: 0755 + - name: manage repos include: repo.yaml loop: "{{ restic_repos | default([]) }}" diff --git a/roles/restic/tasks/repo.yaml b/roles/restic/tasks/repo.yaml index d63fa47..2049288 100644 --- a/roles/restic/tasks/repo.yaml +++ b/roles/restic/tasks/repo.yaml @@ -1,7 +1,7 @@ --- - name: get repo status shell: - cmd: restic -q -r {{ item.repo }} snapshots 2> /dev/null + cmd: restic -q -r {{ item.repo }} --no-lock snapshots 2> /dev/null ignore_errors: yes environment: "{{ item.environment | default({}) }}" register: restic_init @@ -30,8 +30,17 @@ - name: create repo environment helper template: - src: env.sh.j2 + src: repo-env.sh.j2 dest: "{{ restic_etc_path }}/repos/{{ item.name }}/env.sh" owner: root group: root mode: 0400 + +- name: create cron + cron: + name: "restic {{ item.name }} tidy" + hour: "0" + minute: "{{ 60 | random(seed=inventory_hostname) }}" + user: root + state: present + job: "{{ restic_bin_path }}/restic-tidy {{ item.name }}" diff --git a/roles/restic/templates/env.sh.j2 b/roles/restic/templates/env.sh.j2 deleted file mode 100644 index 4cae8cf..0000000 --- a/roles/restic/templates/env.sh.j2 +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -export RESTIC_REPOSITORY="{{ item.repo }}" -{% for k, v in item.environment.items() %} -export {{ k | upper }}="{{ v }}" -{% endfor %} diff --git a/roles/restic/templates/exclude.txt.j2 b/roles/restic/templates/exclude.txt.j2 index b7962d7..cda768d 100644 --- a/roles/restic/templates/exclude.txt.j2 +++ b/roles/restic/templates/exclude.txt.j2 @@ -1,3 +1,15 @@ +# {{ ansible_managed }} + +{% if restic_global_exclude is defined %} +# global exclusions +{% for exclude in restic_global_exclude | default([]) %} +{{ exclude }} +{% endfor %} +{% endif %} + +{% if item.exclude is defined %} +# job exclusions {% for exclude in item.exclude %} {{ exclude }} {% endfor %} +{%- endif -%} diff --git a/roles/restic/templates/job-env.sh.j2 b/roles/restic/templates/job-env.sh.j2 new file mode 100644 index 0000000..545b04e --- /dev/null +++ b/roles/restic/templates/job-env.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/bash + +# {{ ansible_managed }} + +REPO="{{ item.repo }}" +PATHS="{{ item.paths | join(' ') }}" diff --git a/roles/restic/templates/repo-env.sh.j2 b/roles/restic/templates/repo-env.sh.j2 new file mode 100644 index 0000000..e41aa40 --- /dev/null +++ b/roles/restic/templates/repo-env.sh.j2 @@ -0,0 +1,16 @@ +#!/bin/bash + +# {{ ansible_managed }} + +export RESTIC_REPOSITORY="{{ item.repo }}" +{% if "environment" in item %} +{% for k, v in item.environment.items() %} +export {{ k | upper }}="{{ v }}" +{% endfor %} +{% endif %} + +{% if "forget" in item %} +{% for k, v in item.forget.items() %} +{{ k | upper }}="{{ v }}" +{% endfor %} +{% endif %} diff --git a/roles/restic/templates/restic-job.sh.j2 b/roles/restic/templates/restic-job.sh.j2 new file mode 100644 index 0000000..3de307e --- /dev/null +++ b/roles/restic/templates/restic-job.sh.j2 @@ -0,0 +1,50 @@ +#!/bin/bash + +# {{ ansible_managed }} + +error_exit() { + printf "%s\n" "$1" + exit 1 +} + +NICE="ionice -c2 nice -n19" +JOB=$1 + +if [ -z "$JOB" ]; then + error_exit "job is missing" +fi + +RESTIC_ETC_PATH="{{ restic_etc_path }}" +JOB_PATH="${RESTIC_ETC_PATH}/jobs/${JOB}" +JOB_ENV="${JOB_PATH}/env.sh" + +if [ ! -r "$JOB_ENV" ]; then + error_exit "${JOB_ENV} does not exist" +fi + +. "$JOB_ENV" + +if [ -z "${REPO+x}" ]; then + error_exit "\$REPO is not set" +fi + +REPO_PATH="${RESTIC_ETC_PATH}/repos/${REPO}" +REPO_ENV="${REPO_PATH}/env.sh" + +if [ ! -r "$REPO_ENV" ]; then + error_exit "${REPO_ENV} does not exist" +fi + +. "$REPO_ENV" + +EXCLUDE_PATH="${JOB_PATH}/exclude.txt" + +if [ -z "${PATHS+x}" ]; then + error_exit "\$PATHS is not set" +fi + +if [ -r "$EXCLUDE_PATH" ]; then + $NICE {{ restic_path }} backup -q --exclude-file="${EXCLUDE_PATH}" "${PATHS}" +else + $NICE {{ restic_path }} backup -q "${PATHS}" +fi diff --git a/roles/restic/templates/restic-tidy.sh.j2 b/roles/restic/templates/restic-tidy.sh.j2 new file mode 100644 index 0000000..058e2b8 --- /dev/null +++ b/roles/restic/templates/restic-tidy.sh.j2 @@ -0,0 +1,59 @@ +#!/bin/bash + +# {{ ansible_managed }} + +error_exit() { + printf "%s\n" "$1" + exit 1 +} + +MAX_ATTEMPTS=60 +REPO=$1 + +if [ -z "$REPO" ]; then + error_exit "repo is missing" +fi + +RESTIC_ETC_PATH="{{ restic_etc_path }}" +REPO_PATH="${RESTIC_ETC_PATH}/repos/${REPO}" +REPO_ENV="${REPO_PATH}/env.sh" + +if [ ! -r "$REPO_ENV" ]; then + error_exit "${REPO_ENV} does not exist" +fi + +. "$REPO_ENV" + +KEEP_DAILY=${KEEP_DAILY:-7} +KEEP_WEEKLY=${KEEP_WEEKLY:-5} +KEEP_MONTHLY=${KEEP_MONTHLY:-12} +KEEP_YEARLY=${KEEP_YEARLY:-10} + +counter=0 +sleep=1 +rc=1 + +until [ $counter -eq $MAX_ATTEMPTS ] || [ $rc -eq 0 ]; do + {{ restic_path }} forget \ + --quiet \ + --host $(hostname -f) \ + --keep-daily "$KEEP_DAILY" \ + --keep-weekly "$KEEP_WEEKLY" \ + --keep-monthly "$KEEP_MONTHLY" \ + --keep-yearly "$KEEP_YEARLY" \ + --prune + + rc=$? + + if [ $rc -ne 0 ]; then + sleep=$((counter * 5)) + printf "sleeping for %d seconds (%d)\n" $sleep $counter + sleep $sleep + fi + + let counter+=1 +done + +if [ $rc -ne 0 ] && [ $counter -eq $MAX_ATTEMPTS ]; then + printf "tidy timed out, exiting\n" +fi