Perform daily backups. Close #22

restic.yml

@@ -0,0 +1,8 @@
+---
+- hosts:
+    - mail
+
+  become: yes
+
+  roles:
+    - restic

roles/restic/handlers/main.yml

@@ -0,0 +1,11 @@
+---
+- name: Reload s6-rc
+  service:
+    name: s6-rc
+    state: reloaded
+
+- name: Restart restic
+  command: s6-svc -wU -T 5000 -ru /run/service/restic
+
+- name: Restart restic-log
+  command: s6-svc -wU -T 5000 -ru /run/service/restic-log

roles/restic/tasks/main.yml

@@ -0,0 +1,69 @@
+---
+- name: Install restic and snooze
+  package:
+    name:
+      - restic
+      - snooze
+    state: present
+  notify:
+    - Restart restic
+
+- name: Create restic service directories
+  file:
+    path: '/etc/s6-rc/service/{{ item }}'
+    state: directory
+    owner: root
+    group: wheel
+    mode: 0755
+  with_items: '{{ restic_service_dirs }}'
+  notify:
+    - Reload s6-rc
+    - Restart restic
+    - Restart restic-log
+
+- name: Generate restic service scripts
+  template:
+    dest: '/etc/s6-rc/service/{{ item }}'
+    src: '{{ item }}.j2'
+    mode: 0555
+    owner: root
+    group: wheel
+  with_items: '{{ restic_service_scripts }}'
+  notify:
+    - Reload s6-rc
+    - Restart restic
+    - Restart restic-log
+
+- name: Generate restic service configuration
+  copy:
+    dest: '/etc/s6-rc/service/{{ item.name }}'
+    content: '{{ item.content }}'
+    mode: 0444
+    owner: root
+    group: wheel
+  loop_control:
+    label: '{{ item.name }} = {{ item.content }}'
+  with_items: '{{ restic_service_config }}'
+  notify:
+    - Reload s6-rc
+    - Restart restic
+    - Restart restic-log
+
+- name: Flush handlers
+  meta: flush_handlers
+
+- name: Start restic renew service
+  command: fdmove -c 2 1 s6-rc -u -v 2 change restic
+  register: change
+  changed_when: change.stdout | length > 0
+
+- name: Enable restic
+  lineinfile:
+    path: /etc/s6-rc/service/enabled/contents
+    regexp: "^restic$"
+    line: "restic"
+  notify:
+    - Reload s6-rc
+
+- name: Flush handlers (again)
+  meta: flush_handlers

roles/restic/templates/restic-log/finish.j2

@@ -0,0 +1,13 @@
+#!/usr/local/bin/execlineb -S2
+# {{ ansible_managed }}
+
+s6-envdir ./env
+multisubstitute {
+        importas -i -u NAME NAME
+}
+
+fdmove -c 1 2
+ifelse { test "${1}" -eq 0 } {
+        echo "${NAME}: Stopped."
+}
+        echo "${NAME}: Failed with exit status (${1}, ${2})."

roles/restic/templates/restic-log/run.j2

@@ -0,0 +1,23 @@
+#!/usr/local/bin/execlineb -P
+# {{ ansible_managed }}
+
+s6-envdir ./env
+multisubstitute {
+        importas -i -u NAME  NAME
+        importas -i -u USER  USER
+        importas -i -u GROUP GROUP
+        importas -i -u MODE  MODE
+        importas -i -u DIR   DIR
+}
+
+foreground { fdmove -c 1 2 echo "${NAME} log: Starting." }
+
+ifelse -n { install -d -o "${USER}" -g "${GROUP}" -m "${MODE}" "$DIR" } {
+        foreground { fdmove -c 1 2 echo "${NAME} log: Failed to create logging directory." }
+        false
+}
+
+fdmove -c 2 1
+
+s6-envuidgid $USER
+s6-log -d 3 T $DIR

roles/restic/templates/restic/data/job.j2

@@ -0,0 +1,97 @@
+#!/bin/sh
+# {{ ansible_managed }}
+
+# Configure restic through environment variables.
+export RESTIC_PASSWORD_FILE="/root/.restic.pass"
+export RESTIC_REPOSITORY_FILE="/root/.restic.repo"
+export GOMAXPROCS=3
+
+indent() {
+	sed 's/^/        /'
+}
+
+# Ask the kernel for a list of all ZFS filesystems.
+FS_LIST="$(zfs list -H -t filesystem -o name)"
+
+# Back up all mounted ZFS file systems with restic one at a time.
+echo "Backing up ZFS filesystems:"
+for FS in $FS_LIST
+do
+	echo # Seperate the file systems by empty lines (for human consumption)
+
+	# Because scripts shouldn't meddle in the system configuration we
+	# have to skip over unmounted file systems.
+	MNT="$(zfs get -H -t filesystem -o value mountpoint "$FS")"
+	case "$MNT" in
+		/*)
+			echo "* The file system \"$FS\" is mounted at \"$MNT\"."
+			;;
+		*)
+			echo "* Skipping \"$FS\" (no valid mountpoint)."
+			continue
+			;;
+	esac
+	if [ "$(zfs get -H -o value mounted "$FS")" = "no" ]
+	then
+		echo "* Skipping \"$FS\" (not mounted)."
+		continue
+	fi
+
+	# Clean up if the previous run failed
+	if [ -e "$MNT/.zfs/snapshot/restic" ]
+	then
+		echo "* Deleting stale restic snapshot of \"$FS\"."
+		zfs destroy -v "$FS@restic" 2>&1 | indent
+	fi
+
+	# It's important to backup from a snapshot to get a consistent backup.
+	echo "* Taking ZFS snapshot of \"$FS\"."
+	zfs snapshot "$FS@restic" 2>&1 | indent
+
+	# Finally backup the filesystem to a (remote) restic repo.
+	echo "* Backing up \"$FS\"."
+	nice -n 15 restic backup "$MNT/.zfs/snapshot/restic" 2>&1 | indent
+
+	# Clean up the ZFS snapshot.
+	echo "* Deleting ZFS snapshot of \"$FS\"."
+	zfs destroy -v "$FS@restic" 2>&1 | indent
+done
+
+echo
+echo
+
+# Ask the kernel for a list of all ZFS volumes.
+VOL_LIST="$(zfs list -H -t volume -o name)"
+
+echo "Backing up ZFS volumes:"
+for VOL in $VOL_LIST
+do
+	echo
+
+	if [ -e "/dev/zvol/$VOL@restic" ]
+	then
+		echo "* Deleting stale restic snapshot of \"$VOL\"."
+		zfs destroy -v "$VOL@restic" 2>&1 | indent
+	fi
+
+	echo "* Taking ZFS snapshot of \"$VOL\"."
+	zfs snapshot "$VOL@restic" 2>&1 | indent
+
+	echo "* Backing up \"$VOL\"."
+	buffer -s128k -m 128m -i "/dev/zvol/$VOL@restic" | nice -n 15 restic backup --stdin --stdin-filename "$VOL" 2>&1 | indent
+
+	echo "$ Deleting ZFS snapshot of \"$VOL\"."
+	zfs destroy -v "$VOL@restic" 2>&1 | indent
+done
+
+
+echo
+echo
+echo
+
+# Delete old backups
+echo "* Thin out older backups."
+restic forget --keep-yearly 3 --keep-monthly 12 --keep-weekly 13 --keep-daily 31 2>&1 | indent
+
+echo "* Prune unreferenced data from the repository."
+restic prune 2>&1 | indent

roles/restic/templates/restic/finish.j2

@@ -0,0 +1,14 @@
+#!/usr/local/bin/execlineb -S2
+# {{ ansible_managed }}
+
+s6-envdir ./env
+multisubstitute {
+        importas -i -u NAME NAME
+}
+
+fdmove -c 1 2
+ifelse { test "${1}" -eq 0 } {
+	echo "${NAME}: Stopped."
+}
+
+echo "${NAME}: Failed with exit status (${1}, ${2})."

roles/restic/templates/restic/run.j2

@@ -0,0 +1,12 @@
+#!/usr/local/bin/execlineb -P
+# {{ ansible_managed }}
+
+s6-envdir ./env
+multisubstitute {
+        importas -i -u NAME NAME
+}
+
+foreground { fdmove -c 1 2 echo "${NAME}: Starting." }
+
+fdmove -c 2 1
+snooze -v data/job

roles/restic/vars/main.yml

@@ -0,0 +1,47 @@
+---
+restic_service_dirs:
+  - restic
+  - restic/env
+  - restic/data
+  - restic-log
+  - restic-log/env
+
+restic_service_scripts:
+  - restic/run
+  - restic/finish
+  - restic/data/job
+  - restic-log/run
+  - restic-log/finish
+
+restic_service_config:
+  - name: restic/type
+    content: longrun
+  - name: restic/dependencies
+    content: postfix
+  - name: restic/env/HOME
+    content: /root
+  - name: restic/env/NAME
+    content: restic
+  - name: restic/env/PATH
+    content: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/root/bin
+  - name: restic/producer-for
+    content: restic-log
+
+  - name: restic-log/type
+    content: longrun
+  - name: restic-log/notification-fd
+    content: '3'
+  - name: restic-log/consumer-for
+    content: restic
+  - name: restic-log/env/NAME
+    content: restic
+  - name: restic-log/env/PATH
+    content: /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/root/bin
+  - name: restic-log/env/MODE
+    content: '750'
+  - name: restic-log/env/USER
+    content: s6-log
+  - name: restic-log/env/GROUP
+    content: s6-log
+  - name: restic-log/env/DIR
+    content: /var/log/restic

site.yml

@@ -5,3 +5,4 @@
 - import_playbook: haproxy.yml
 - import_playbook: bhyve.yml
 - import_playbook: mail.yml
+- import_playbook: restic.yml