From 6982d0feaae84d110dac84b2c745baeaadda9d53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reynir=20Bj=C3=B6rnsson?= Date: Sun, 3 Mar 2024 21:17:48 +0100 Subject: [PATCH] Restic: send an email on backup failure --- roles/docker/defaults/main.yml | 4 ++++ roles/docker/tasks/pre_deploy/restic.yml | 16 ++++++++++++++++ .../docker/templates/compose-files/restic.yml.j2 | 9 +++++++++ roles/docker/templates/restic/failure.sh.j2 | 14 ++++++++++++++ 4 files changed, 43 insertions(+) create mode 100644 roles/docker/templates/restic/failure.sh.j2 diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 4ba6a88e..dcb58b22 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -50,6 +50,10 @@ services: repository: restic version: "1.7.0" disabled_in_vagrant: true + # mail dance + domain: "noreply.{{ base_domain }}" + allowed_sender_domain: true + mail-from: "noreply@noreply.{{ base_domain }}" docker_registry: domain: "docker.{{ base_domain }}" diff --git a/roles/docker/tasks/pre_deploy/restic.yml b/roles/docker/tasks/pre_deploy/restic.yml index 8a147d77..1c32e907 100644 --- a/roles/docker/tasks/pre_deploy/restic.yml +++ b/roles/docker/tasks/pre_deploy/restic.yml @@ -46,3 +46,19 @@ owner: root group: root mode: '0600' + +- name: Create scripts directory + file: + path: "{{ services.restic.volume_folder }}/scripts" + owner: root + group: root + mode: '0755' + state: directory + +- name: Upload failure.sh script + template: + src: restic/failure.sh.j2 + dest: "{{ services.restic.volume_folder }}/scripts/failure.sh" + owner: root + group: root + mode: '0755' diff --git a/roles/docker/templates/compose-files/restic.yml.j2 b/roles/docker/templates/compose-files/restic.yml.j2 index 333a7e5b..b69571e6 100644 --- a/roles/docker/templates/compose-files/restic.yml.j2 +++ b/roles/docker/templates/compose-files/restic.yml.j2 @@ -15,6 +15,7 @@ services: --tag datacoop-volumes --exclude '*.tmp' --exclude '/mnt/volumes/mastodon/mastodon_data/cache/' + --exclude '/mnt/volumes/restic/' --verbose RESTIC_FORGET_ARGS: >- --keep-last 10 @@ -22,9 +23,13 @@ services: --keep-weekly 5 --keep-monthly 12 TZ: Europe/Copenhagen + POST_COMMANDS_FAILURE=/run/libexec/failure.sh volumes: - "./ssh:/run/secrets/.ssh:ro" + - "./scripts:/run/libexec:ro" - "/docker-volumes:/mnt/volumes:ro" + networks: + - postfix prune: image: mazzolino/restic:{{ services.restic.version }} @@ -36,3 +41,7 @@ services: TZ: Europe/copenhagen volumes: - "./ssh:/run/secrets/.ssh:ro" + +networks: + postfix: + external: true diff --git a/roles/docker/templates/restic/failure.sh.j2 b/roles/docker/templates/restic/failure.sh.j2 new file mode 100644 index 00000000..5f8411d2 --- /dev/null +++ b/roles/docker/templates/restic/failure.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/sh +curl smtp://postfix --mail-from {{ services.restic.mail-from }} --mail-rcpt admin-hold@data.coop --upload-file . << END_OF_MAIL +From: Restic backup <{{ services.restic.mail-from }}> +To: admin-hold@data.coop +Subject: Restic backup failed +Date: $(date) + +Dear sir or madam, + +Tonight's backup failed! + +Best, +Your backup software. +END_OF_MAIL