From 6982d0feaae84d110dac84b2c745baeaadda9d53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Reynir=20Bj=C3=B6rnsson?= <reynir@reynir.dk>
Date: Sun, 3 Mar 2024 21:17:48 +0100
Subject: [PATCH] Restic: send an email on backup failure

---
 roles/docker/defaults/main.yml                   |  4 ++++
 roles/docker/tasks/pre_deploy/restic.yml         | 16 ++++++++++++++++
 .../docker/templates/compose-files/restic.yml.j2 |  9 +++++++++
 roles/docker/templates/restic/failure.sh.j2      | 14 ++++++++++++++
 4 files changed, 43 insertions(+)
 create mode 100644 roles/docker/templates/restic/failure.sh.j2

diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml
index 4ba6a88..dcb58b2 100644
--- a/roles/docker/defaults/main.yml
+++ b/roles/docker/defaults/main.yml
@@ -50,6 +50,10 @@ services:
     repository: restic
     version: "1.7.0"
     disabled_in_vagrant: true
+    # mail dance
+    domain: "noreply.{{ base_domain }}"
+    allowed_sender_domain: true
+    mail-from: "noreply@noreply.{{ base_domain }}"
 
   docker_registry:
     domain: "docker.{{ base_domain }}"
diff --git a/roles/docker/tasks/pre_deploy/restic.yml b/roles/docker/tasks/pre_deploy/restic.yml
index 8a147d7..1c32e90 100644
--- a/roles/docker/tasks/pre_deploy/restic.yml
+++ b/roles/docker/tasks/pre_deploy/restic.yml
@@ -46,3 +46,19 @@
     owner: root
     group: root
     mode: '0600'
+
+- name: Create scripts directory
+  file:
+    path: "{{ services.restic.volume_folder }}/scripts"
+    owner: root
+    group: root
+    mode: '0755'
+    state: directory
+
+- name: Upload failure.sh script
+  template:
+    src: restic/failure.sh.j2
+    dest: "{{ services.restic.volume_folder }}/scripts/failure.sh"
+    owner: root
+    group: root
+    mode: '0755'
diff --git a/roles/docker/templates/compose-files/restic.yml.j2 b/roles/docker/templates/compose-files/restic.yml.j2
index 333a7e5..b69571e 100644
--- a/roles/docker/templates/compose-files/restic.yml.j2
+++ b/roles/docker/templates/compose-files/restic.yml.j2
@@ -15,6 +15,7 @@ services:
         --tag datacoop-volumes
         --exclude '*.tmp'
         --exclude '/mnt/volumes/mastodon/mastodon_data/cache/'
+	--exclude '/mnt/volumes/restic/'
         --verbose
       RESTIC_FORGET_ARGS: >-
         --keep-last 10
@@ -22,9 +23,13 @@ services:
         --keep-weekly 5
         --keep-monthly 12
       TZ: Europe/Copenhagen
+      POST_COMMANDS_FAILURE=/run/libexec/failure.sh
     volumes:
       - "./ssh:/run/secrets/.ssh:ro"
+      - "./scripts:/run/libexec:ro"
       - "/docker-volumes:/mnt/volumes:ro"
+    networks:
+      - postfix
 
   prune:
     image: mazzolino/restic:{{ services.restic.version }}
@@ -36,3 +41,7 @@ services:
       TZ: Europe/copenhagen
     volumes:
       - "./ssh:/run/secrets/.ssh:ro"
+
+networks:
+  postfix:
+    external: true
diff --git a/roles/docker/templates/restic/failure.sh.j2 b/roles/docker/templates/restic/failure.sh.j2
new file mode 100644
index 0000000..5f8411d
--- /dev/null
+++ b/roles/docker/templates/restic/failure.sh.j2
@@ -0,0 +1,14 @@
+#!/bin/sh
+curl smtp://postfix --mail-from {{ services.restic.mail-from }} --mail-rcpt admin-hold@data.coop --upload-file . << END_OF_MAIL
+From: Restic backup <{{ services.restic.mail-from }}>
+To: admin-hold@data.coop
+Subject: Restic backup failed
+Date: $(date)
+
+Dear sir or madam,
+
+Tonight's backup failed!
+
+Best,
+Your backup software.
+END_OF_MAIL