From a47440b6b5a213f4c7db2b11efc72d62948e1b6c Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 30 Sep 2023 16:25:06 +0200 Subject: [PATCH 01/23] Move compose files into templates and upload them to the host --- roles/docker/defaults/main.yml | 2 +- roles/docker/tasks/services/drone.yml | 58 +----- roles/docker/tasks/services/hedgedoc.yml | 55 +----- roles/docker/tasks/services/keycloak.yml | 56 +----- roles/docker/tasks/services/mailu.yml | 147 ++------------- roles/docker/tasks/services/mastodon.yml | 167 ++---------------- .../docker/tasks/services/matrix_element.yml | 66 +------ roles/docker/tasks/services/membersystem.yml | 58 +----- roles/docker/tasks/services/nextcloud.yml | 76 +------- roles/docker/tasks/services/passit.yml | 52 +----- roles/docker/tasks/services/rallly.yml | 59 ++----- roles/docker/tasks/services/restic.yml | 59 +++++++ roles/docker/tasks/services/restic_backup.yml | 89 ---------- .../templates/compose-files/drone.yml.j2 | 40 +++++ .../templates/compose-files/hedgedoc.yml.j2 | 44 +++++ .../templates/compose-files/keycloak.yml.j2 | 42 +++++ .../templates/compose-files/mailu.yml.j2 | 131 ++++++++++++++ .../templates/compose-files/mastodon.yml.j2 | 146 +++++++++++++++ .../compose-files/matrix_element.yml.j2 | 52 ++++++ .../compose-files/membersystem.yml.j2 | 44 +++++ .../templates/compose-files/nextcloud.yml.j2 | 59 +++++++ .../templates/compose-files/passit.yml.j2 | 38 ++++ .../templates/compose-files/rallly.yml.j2 | 41 +++++ .../templates/compose-files/restic.yml.j2 | 37 ++++ roles/docker/templates/rallly/env.j2 | 2 +- 25 files changed, 827 insertions(+), 793 deletions(-) create mode 100644 roles/docker/tasks/services/restic.yml delete mode 100644 roles/docker/tasks/services/restic_backup.yml create mode 100644 roles/docker/templates/compose-files/drone.yml.j2 create mode 100644 roles/docker/templates/compose-files/hedgedoc.yml.j2 create mode 100644 roles/docker/templates/compose-files/keycloak.yml.j2 create mode 100644 roles/docker/templates/compose-files/mailu.yml.j2 create mode 100644 roles/docker/templates/compose-files/mastodon.yml.j2 create mode 100644 roles/docker/templates/compose-files/matrix_element.yml.j2 create mode 100644 roles/docker/templates/compose-files/membersystem.yml.j2 create mode 100644 roles/docker/templates/compose-files/nextcloud.yml.j2 create mode 100644 roles/docker/templates/compose-files/passit.yml.j2 create mode 100644 roles/docker/templates/compose-files/rallly.yml.j2 create mode 100644 roles/docker/templates/compose-files/restic.yml.j2 diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 11979f40..5477414c 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -48,7 +48,7 @@ services: allowed_sender_domain: true restic: - file: restic_backup.yml + file: restic.yml user: dc-user domain: rynkeby.skovgaard.tel host_key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBLGol2G+a87ssy0nu/STKBZSiGyhZhZKx/ujfe9IeFo diff --git a/roles/docker/tasks/services/drone.yml b/roles/docker/tasks/services/drone.yml index 22b71cc7..de8720e5 100644 --- a/roles/docker/tasks/services/drone.yml +++ b/roles/docker/tasks/services/drone.yml @@ -1,52 +1,12 @@ # vim: ft=yaml.ansible --- -- name: set up drone with docker runner +- name: Upload Compose file for Drone + template: + src: compose-files/drone.yml.j2 + dest: "{{ services.drone.volume_folder }}/docker-compose.yml" + +- name: Deploy Drone docker_compose: - project_name: drone - pull: yes - definition: - version: "3.6" - services: - drone: - container_name: "drone" - image: "drone/drone:{{ services.drone.version }}" - restart: unless-stopped - networks: - - external_services - - drone - volumes: - - "{{ services.drone.volume_folder }}:/data" - - "/var/run/docker.sock:/var/run/docker.sock" - environment: - DRONE_GITEA_SERVER: "https://{{ services.forgejo.domain }}" - DRONE_GITEA_CLIENT_ID: "{{ drone_secrets.oauth_client_id }}" - DRONE_GITEA_CLIENT_SECRET: "{{ drone_secrets.oauth_client_secret }}" - DRONE_GIT_ALWAYS_AUTH: "true" - DRONE_SERVER_HOST: "{{ services.drone.domain }}" - DRONE_SERVER_PROTO: "https" - DRONE_RPC_SECRET: "{{ drone_secrets.rpc_shared_secret }}" - PLUGIN_CUSTOM_DNS: "91.239.100.100" - VIRTUAL_HOST: "{{ services.drone.domain }}" - LETSENCRYPT_HOST: "{{ services.drone.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - - drone-runner-docker: - container_name: "drone-runner-docker" - image: "drone/drone-runner-docker:{{ services.drone.version }}" - restart: unless-stopped - networks: - - drone - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - environment: - DRONE_RPC_HOST: "{{ services.drone.domain }}" - DRONE_RPC_PROTO: "https" - DRONE_RPC_SECRET: "{{ drone_secrets.rpc_shared_secret }}" - DRONE_RUNNER_CAPACITY: 2 - DRONE_RUNNER_NAME: "data.coop_drone_runner" - - networks: - drone: - external_services: - external: - name: external_services + project_src: "{{ services.drone.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/hedgedoc.yml b/roles/docker/tasks/services/hedgedoc.yml index 8160a66c..6e5c874a 100644 --- a/roles/docker/tasks/services/hedgedoc.yml +++ b/roles/docker/tasks/services/hedgedoc.yml @@ -16,52 +16,13 @@ dest: "{{ services.hedgedoc.volume_folder }}/sso.data.coop.pem" mode: "0644" +- name: Upload Compose file for for HedgeDoc + template: + src: compose-files/hedgedoc.yml.j2 + dest: "{{ services.hedgedoc.volume_folder }}/docker-compose.yml" + - name: setup hedgedoc docker_compose: - project_name: "hedgedoc" - pull: "yes" - definition: - services: - database: - image: "postgres:{{ services.hedgedoc.postgres_version }}" - environment: - POSTGRES_USER: "codimd" - POSTGRES_PASSWORD: "{{ postgres_passwords.hedgedoc }}" - POSTGRES_DB: "codimd" - restart: "unless-stopped" - networks: - - "hedgedoc" - volumes: - - "{{ services.hedgedoc.volume_folder }}/db:/var/lib/postgresql/data" - - app: - image: "quay.io/hedgedoc/hedgedoc:{{ services.hedgedoc.version }}" - environment: - CMD_DB_URL: "postgres://codimd:{{ postgres_passwords.hedgedoc }}@hedgedoc_database_1:5432/codimd" - CMD_DOMAIN: "{{ services.hedgedoc.domain }}" - CMD_ALLOW_EMAIL_REGISTER: "False" - CMD_IMAGE_UPLOAD_TYPE: "filesystem" - CMD_EMAIL: "False" - CMD_SAML_IDPCERT: "/sso.data.coop.pem" - CMD_SAML_IDPSSOURL: "https://sso.data.coop/auth/realms/datacoop/protocol/saml" - CMD_SAML_ISSUER: "hedgedoc" - CMD_SAML_IDENTIFIERFORMAT: "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified" - CMD_USECDN: "false" - CMD_PROTOCOL_USESSL: "true" - VIRTUAL_HOST: "{{ services.hedgedoc.domain }}" - LETSENCRYPT_HOST: "{{ services.hedgedoc.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - volumes: - - "{{ services.hedgedoc.volume_folder }}/hedgedoc/uploads:/hedgedoc/public/uploads" - - "{{ services.hedgedoc.volume_folder }}/sso.data.coop.pem:/sso.data.coop.pem" - restart: "unless-stopped" - networks: - - "hedgedoc" - - "external_services" - depends_on: - - database - - networks: - hedgedoc: - external_services: - external: true + project_src: "{{ services.hedgedoc.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/keycloak.yml b/roles/docker/tasks/services/keycloak.yml index 7c23cfdd..45feb25b 100644 --- a/roles/docker/tasks/services/keycloak.yml +++ b/roles/docker/tasks/services/keycloak.yml @@ -1,50 +1,12 @@ # vim: ft=yaml.ansible --- -- name: setup keycloak containers for sso.data.coop +- name: Upload Compose file for for Keycloak + template: + src: compose-files/keycloak.yml.j2 + dest: "{{ services.keycloak.volume_folder }}/docker-compose.yml" + +- name: Deploy Keycloak docker_compose: - project_name: "keycloak" - pull: "yes" - definition: - version: "3.6" - services: - postgres: - image: "postgres:{{ services.keycloak.postgres_version }}" - restart: "unless-stopped" - networks: - - "keycloak" - volumes: - - "{{ services.keycloak.volume_folder }}/data:/var/lib/postgresql/data" - environment: - POSTGRES_USER: "keycloak" - POSTGRES_PASSWORD: "{{ postgres_passwords.keycloak }}" - POSTGRES_DB: "keycloak" - - app: - image: "quay.io/keycloak/keycloak:{{ services.keycloak.version }}" - restart: "unless-stopped" - networks: - - "keycloak" - - "postfix" - - "external_services" - command: - - "start" - - "--db=postgres" - - "--db-url=jdbc:postgresql://postgres:5432/keycloak" - - "--db-username=keycloak" - - "--db-password={{ postgres_passwords.keycloak }}" - - "--hostname={{ services.keycloak.domain }}" - - "--proxy=edge" - - "--https-port=8080" - - "--http-relative-path=/auth" - environment: - VIRTUAL_HOST: "{{ services.keycloak.domain }}" - VIRTUAL_PORT: "8080" - LETSENCRYPT_HOST: "{{ services.keycloak.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - - networks: - keycloak: - postfix: - external: true - external_services: - external: true + project_src: "{{ services.keycloak.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/mailu.yml b/roles/docker/tasks/services/mailu.yml index 168609a0..de4916d9 100644 --- a/roles/docker/tasks/services/mailu.yml +++ b/roles/docker/tasks/services/mailu.yml @@ -33,7 +33,7 @@ src: "{{ services.nginx_proxy.volume_folder }}/certs/{{ services.mailu.domain }}/fullchain.pem" dest: "{{ services.mailu.volume_folder }}/certs/cert.pem" state: hard - force: yes + force: true when: letsencrypt_enabled - name: hard link to Let's Encrypt TLS key @@ -41,141 +41,16 @@ src: "{{ services.nginx_proxy.volume_folder }}/certs/{{ services.mailu.domain }}/key.pem" dest: "{{ services.mailu.volume_folder }}/certs/key.pem" state: hard - force: yes + force: true when: letsencrypt_enabled -- name: run mail server containers +- name: Upload Compose file for for Mailu + template: + src: compose-files/mailu.yml.j2 + dest: "{{ services.mailu.volume_folder }}/docker-compose.yml" + +- name: Deploy Mailu docker_compose: - project_name: mail_server - pull: yes - definition: - version: '3.6' - services: - postgres: - image: postgres:14-alpine - restart: always - environment: - POSTGRES_DB: mailu - POSTGRES_USER: mailu - POSTGRES_PASSWORD: "{{ postgres_passwords.mailu }}" - volumes: - - "{{ services.mailu.volume_folder }}/postgres:/var/lib/postgresql/data" - dns: - - "{{ services.mailu.dns }}" - - redis: - image: redis:alpine - restart: always - volumes: - - "{{ services.mailu.volume_folder }}/redis:/data" - depends_on: - - resolver - dns: - - "{{ services.mailu.dns }}" - - front: - image: ghcr.io/mailu/nginx:{{ services.mailu.version }} - restart: always - env_file: "{{ services.mailu.volume_folder }}/mailu.env" - environment: - VIRTUAL_HOST: "{{ services.mailu.domain }}" - LETSENCRYPT_HOST: "{{ services.mailu.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - volumes: - - "{{ services.mailu.volume_folder }}/certs:/certs" - - "{{ services.mailu.volume_folder }}/overrides/nginx:/overrides:ro" - expose: - - "80" - ports: - - "993:993" - - "25:25" - - "587:587" - - "465:465" - networks: - - default - - external_services - - resolver: - image: ghcr.io/mailu/unbound:{{ services.mailu.version }} - restart: always - env_file: "{{ services.mailu.volume_folder }}/mailu.env" - networks: - default: - ipv4_address: "{{ services.mailu.dns }}" - - admin: - image: ghcr.io/mailu/admin:{{ services.mailu.version }} - restart: always - env_file: "{{ services.mailu.volume_folder }}/mailu.env" - volumes: - - "{{ services.mailu.volume_folder }}/data:/data" - - "{{ services.mailu.volume_folder }}/dkim:/dkim" - depends_on: - - redis - - resolver - dns: - - "{{ services.mailu.dns }}" - - imap: - image: ghcr.io/mailu/dovecot:{{ services.mailu.version }} - restart: always - env_file: "{{ services.mailu.volume_folder }}/mailu.env" - volumes: - - "{{ services.mailu.volume_folder }}/mail:/mail" - - "{{ services.mailu.volume_folder }}/overrides/dovecot:/overrides:ro" - depends_on: - - front - - resolver - dns: - - "{{ services.mailu.dns }}" - - smtp: - image: ghcr.io/mailu/postfix:{{ services.mailu.version }} - restart: always - env_file: "{{ services.mailu.volume_folder }}/mailu.env" - volumes: - - "{{ services.mailu.volume_folder }}/mailqueue:/queue" - - "{{ services.mailu.volume_folder }}/overrides/postfix:/overrides:ro" - depends_on: - - front - - resolver - dns: - - "{{ services.mailu.dns }}" - - antispam: - image: ghcr.io/mailu/rspamd:{{ services.mailu.version }} - hostname: antispam - restart: always - env_file: "{{ services.mailu.volume_folder }}/mailu.env" - volumes: - - "{{ services.mailu.volume_folder }}/filter:/var/lib/rspamd" - - "{{ services.mailu.volume_folder }}/overrides/rspamd:/etc/rspamd/override.d:ro" - depends_on: - - front - - resolver - dns: - - "{{ services.mailu.dns }}" - - webmail: - image: ghcr.io/mailu/rainloop:{{ services.mailu.version }} - restart: always - env_file: "{{ services.mailu.volume_folder }}/mailu.env" - volumes: - - "{{ services.mailu.volume_folder }}/webmail:/data" - - "{{ services.mailu.volume_folder }}/overrides/rainloop:/overrides:ro" - depends_on: - - imap - - resolver - dns: - - "{{ services.mailu.dns }}" - - networks: - default: - driver: bridge - ipam: - driver: default - config: - - subnet: "{{ services.mailu.subnet }}" - external_services: - external: - name: external_services + project_src: "{{ services.mailu.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/mastodon.yml b/roles/docker/tasks/services/mastodon.yml index 1c100cee..95a14c08 100644 --- a/roles/docker/tasks/services/mastodon.yml +++ b/roles/docker/tasks/services/mastodon.yml @@ -32,9 +32,9 @@ - name: Copy mastodon environment file template: src: mastodon/env.j2 - dest: "{{ services.mastodon.volume_folder }}/env_file" + dest: "{{ services.mastodon.volume_folder }}/mastodon.env" -- name: Upload vhost config for root domain +- name: Upload vhost config for Mastodon domain copy: src: vhost/mastodon dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.mastodon.domain }}" @@ -44,164 +44,17 @@ src: mastodon/postgresql.conf dest: "{{ services.mastodon.volume_folder }}/postgres_config/postgresql.conf" -- name: Set up Mastodon +- name: Upload Compose file for Mastodon + template: + src: compose-files/mastodon.yml.j2 + dest: "{{ services.mastodon.volume_folder }}/docker-compose.yml" + +- name: Deploy Mastodon docker_compose: - project_name: mastodon + project_src: "{{ services.mastodon.volume_folder }}" pull: true restarted: true - definition: - x-sidekiq: &sidekiq - image: "tootsuite/mastodon:{{ services.mastodon.version }}" - restart: always - env_file: "{{ services.mastodon.volume_folder }}/env_file" - depends_on: - db: - condition: "service_healthy" - redis: - condition: "service_healthy" - networks: - - postfix - - external_services - - internal_network - volumes: - - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system" - healthcheck: - test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"] - - version: '3' - services: - db: - restart: always - image: "postgres:{{ services.mastodon.postgres_version }}" - shm_size: 256mb - networks: - - internal_network - healthcheck: - test: ['CMD', 'pg_isready', '-U', 'postgres'] - volumes: - - "{{ services.mastodon.volume_folder }}/postgres_data:/var/lib/postgresql/data" - - "{{ services.mastodon.volume_folder }}/postgres_config:/config:ro" - command: postgres -c config_file=/config/postgresql.conf - environment: - - 'POSTGRES_HOST_AUTH_METHOD=trust' - - redis: - restart: always - image: "redis:{{ services.mastodon.redis_version }}" - networks: - - internal_network - healthcheck: - test: ['CMD', 'redis-cli', 'ping'] - volumes: - - "{{ services.mastodon.volume_folder }}/redis_data:/data" - - web: - image: "tootsuite/mastodon:{{ services.mastodon.version }}" - restart: always - env_file: "{{ services.mastodon.volume_folder }}/env_file" - command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" - networks: - - external_services - - internal_network - healthcheck: - # prettier-ignore - test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:3000/health || exit 1'] - depends_on: - db: - condition: "service_healthy" - redis: - condition: "service_healthy" - volumes: - - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system" - environment: - MAX_THREADS: 10 - WEB_CONCURRENCY: 3 - VIRTUAL_HOST: "{{ services.mastodon.domain }}" - VIRTUAL_PORT: "3000" - VIRTUAL_PATH: "/" - LETSENCRYPT_HOST: "{{ services.mastodon.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - - streaming: - image: "tootsuite/mastodon:{{ services.mastodon.version }}" - restart: always - env_file: "{{ services.mastodon.volume_folder }}/env_file" - command: node ./streaming - networks: - - external_services - - internal_network - healthcheck: - # prettier-ignore - test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1'] - ports: - - '127.0.0.1:4000:4000' - depends_on: - db: - condition: "service_healthy" - redis: - condition: "service_healthy" - environment: - DB_POOL: 15 - VIRTUAL_HOST: "{{ services.mastodon.domain }}" - VIRTUAL_PORT: "4000" - VIRTUAL_PATH: "/api/v1/streaming" - - # sidekiq-default-push-pull: DB_POOL = 25, -c 25 for 25 connections - sidekiq-default-push-pull: - <<: *sidekiq - command: bundle exec sidekiq -c 25 -q default -q push -q pull - environment: - DB_POOL: 25 - - # sidekiq-default-pull-push: DB_POOL = 25, -c 25 for 25 connections - sidekiq-default-pull-push: - <<: *sidekiq - command: bundle exec sidekiq -c 25 -q default -q pull -q push - environment: - DB_POOL: 25 - - # sidekiq-pull-default-push: DB_POOL = 25, -c 25 for 25 connections - sidekiq-pull-default-push: - <<: *sidekiq - command: bundle exec sidekiq -c 25 -q pull -q default -q push - environment: - DB_POOL: 25 - - # sidekiq-push-default-pull: DB_POOL = 25, -c 25 for 25 connections - sidekiq-push-default-pull: - <<: *sidekiq - command: bundle exec sidekiq -c 25 -q push -q default -q pull - environment: - DB_POOL: 25 - - # sidekiq-push-scheduler: DB_POOL = 5, -c 5 for 5 connections - sidekiq-push-scheduler: - <<: *sidekiq - command: bundle exec sidekiq -c 5 -q push -q scheduler - environment: - DB_POOL: 5 - - # sidekiq-push-mailers: DB_POOL = 5, -c 5 for 5 connections - sidekiq-push-mailers: - <<: *sidekiq - command: bundle exec sidekiq -c 5 -q push -q mailers - environment: - DB_POOL: 5 - - # sidekiq-push-ingress: DB_POOL = 10, -c 10 for 10 connections - sidekiq-push-ingress: - <<: *sidekiq - command: bundle exec sidekiq -c 10 -q push -q ingress - environment: - DB_POOL: 10 - - networks: - external_services: - external: true - postfix: - external: true - internal_network: - internal: true + state: present - name: Configure cron job to remove old Mastodon media daily cron: diff --git a/roles/docker/tasks/services/matrix_element.yml b/roles/docker/tasks/services/matrix_element.yml index b5a04b0d..605084b3 100644 --- a/roles/docker/tasks/services/matrix_element.yml +++ b/roles/docker/tasks/services/matrix_element.yml @@ -58,63 +58,13 @@ src: matrix/log.config dest: "{{ services.matrix.volume_folder }}/data/matrix.data.coop.log.config" -- name: Set up Matrix and Element +- name: Upload Compose file for Matrix and Element + template: + src: compose-files/matrix_element.yml.j2 + dest: "{{ services.matrix.volume_folder }}/docker-compose.yml" + +- name: Deploy Matrix and Element docker_compose: - project_name: matrix + project_src: "{{ services.matrix.volume_folder }}" pull: true - definition: - version: "3.6" - services: - postgres: - image: "postgres:{{ services.matrix.postgres_version }}" - restart: unless-stopped - networks: - - matrix - volumes: - - "{{ services.matrix.volume_folder }}/db:/var/lib/postgresql/data" - environment: - POSTGRES_USER: "synapse" - POSTGRES_PASSWORD: "{{ postgres_passwords.matrix }}" - - synapse: - image: "matrixdotorg/synapse:{{ services.matrix.version }}" - restart: unless-stopped - networks: - - matrix - - external_services - - postfix - volumes: - - "{{ services.matrix.volume_folder }}/data:/data" - environment: - SYNAPSE_CONFIG_PATH: "/data/homeserver.yaml" - SYNAPSE_CACHE_FACTOR: "2" - SYNAPSE_LOG_LEVEL: "INFO" - VIRTUAL_HOST: "{{ services.matrix.domain }}" - VIRTUAL_PORT: "8008" - LETSENCRYPT_HOST: "{{ services.matrix.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - - element: - image: "avhost/docker-matrix-element:{{ services.element.version }}" - restart: unless-stopped - networks: - - matrix - - external_services - expose: - - 8080 - volumes: - - "{{ services.element.volume_folder }}/data:/data" - environment: - VIRTUAL_HOST: "{{ services.element.domains | join(',') }}" - VIRTUAL_PORT: "8080" - LETSENCRYPT_HOST: "{{ services.element.domains | join(',') }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - - networks: - external_services: - external: - name: external_services - postfix: - external: true - matrix: - name: "matrix" + state: present diff --git a/roles/docker/tasks/services/membersystem.yml b/roles/docker/tasks/services/membersystem.yml index d310554b..ba9135e8 100644 --- a/roles/docker/tasks/services/membersystem.yml +++ b/roles/docker/tasks/services/membersystem.yml @@ -1,52 +1,12 @@ # vim: ft=yaml.ansible --- -- name: run membersystem containers +- name: Upload Compose file for Membersystem + template: + src: compose-files/membersystem.yml.j2 + dest: "{{ services.membersystem.volume_folder }}/docker-compose.yml" + +- name: Deploy Membersystem docker_compose: - project_name: "member.data.coop" - pull: yes - definition: - version: "3" - services: - backend: - image: "docker.data.coop/membersystem:{{ services.membersystem.version }}" - restart: always - user: $UID:$GID - tty: true - depends_on: - - postgres - networks: - - membersystem - - external_services - - postfix - environment: - SECRET_KEY: "{{ membersystem_secrets.secret_key }}" - DATABASE_URL: postgres://postgres:{{ postgres_passwords.membersystem }}@postgres:5432/postgres - POSTGRES_HOST: postgres - POSTGRES_PORT: 5432 - EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend" - EMAIL_URL: "smtp://noop@{{ smtp_host }}:{{ smtp_port }}" - VIRTUAL_HOST: "{{ services.membersystem.domain }}" - VIRTUAL_PORT: "8000" - LETSENCRYPT_HOST: "{{ services.membersystem.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - ALLOWED_HOSTS: "{{ services.membersystem.domain }}" - CSRF_TRUSTED_ORIGINS: "https://{{ services.membersystem.domain }}" - DJANGO_ADMINS: "{{ services.membersystem.django_admins }}" - DEFAULT_FROM_EMAIL: "noreply@{{ services.membersystem.domain }}" - - postgres: - image: "postgres:{{ services.membersystem.postgres_version }}" - restart: always - volumes: - - "{{ volume_root_folder }}/membersystem/postgres/data:/var/lib/postgresql/data" - networks: - - membersystem - environment: - POSTGRES_PASSWORD: "{{ postgres_passwords.membersystem }}" - - networks: - membersystem: - external_services: - external: true - postfix: - external: true + project_src: "{{ services.membersystem.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/nextcloud.yml b/roles/docker/tasks/services/nextcloud.yml index f1d19b0b..acfa587c 100644 --- a/roles/docker/tasks/services/nextcloud.yml +++ b/roles/docker/tasks/services/nextcloud.yml @@ -6,71 +6,13 @@ dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.nextcloud.domain }}" notify: "restart nginx" -- name: setup nextcloud containers +- name: Upload Compose file for Nextcloud + template: + src: compose-files/nextcloud.yml.j2 + dest: "{{ services.nextcloud.volume_folder }}/docker-compose.yml" + +- name: Deploy Nextcloud docker_compose: - project_name: "nextcloud" - pull: "yes" - definition: - services: - postgres: - image: "postgres:{{ services.nextcloud.postgres_version }}" - restart: "unless-stopped" - networks: - - "nextcloud" - volumes: - - "{{ services.nextcloud.volume_folder }}/postgres:/var/lib/postgresql/data" - environment: - POSTGRES_DB: "nextcloud" - POSTGRES_PASSWORD: "{{ postgres_passwords.nextcloud }}" - POSTGRES_USER: "nextcloud" - - redis: - image: "redis:{{ services.nextcloud.redis_version }}" - restart: "unless-stopped" - command: "redis-server --requirepass {{ nextcloud_secrets.redis_password }}" - tmpfs: - - /var/lib/redis - networks: - - "nextcloud" - - cron: - image: "nextcloud:{{ services.nextcloud.version }}" - restart: "unless-stopped" - entrypoint: "/cron.sh" - networks: - - "nextcloud" - volumes: - - "{{ services.nextcloud.volume_folder }}/app:/var/www/html" - depends_on: - - "postgres" - - "redis" - - app: - image: "nextcloud:{{ services.nextcloud.version }}" - restart: "unless-stopped" - networks: - - "nextcloud" - - "postfix" - - "external_services" - volumes: - - "{{ services.nextcloud.volume_folder }}/app:/var/www/html" - environment: - VIRTUAL_HOST: "{{ services.nextcloud.domain }}" - LETSENCRYPT_HOST: "{{ services.nextcloud.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - POSTGRES_HOST: "postgres" - POSTGRES_DB: "nextcloud" - POSTGRES_USER: "nextcloud" - POSTGRES_PASSWORD: "{{ postgres_passwords.nextcloud }}" - REDIS_HOST: "redis" - REDIS_HOST_PASSWORD: "{{ nextcloud_secrets.redis_password }}" - depends_on: - - "postgres" - - "redis" - - networks: - nextcloud: - postfix: - external: true - external_services: - external: true + project_src: "{{ services.nextcloud.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/passit.yml b/roles/docker/tasks/services/passit.yml index 58e87a6b..eaf5baad 100644 --- a/roles/docker/tasks/services/passit.yml +++ b/roles/docker/tasks/services/passit.yml @@ -7,47 +7,13 @@ group: root state: directory -- name: setup passit containers +- name: Upload Compose file for Passit + template: + src: compose-files/passit.yml.j2 + dest: "{{ services.passit.volume_folder }}/docker-compose.yml" + +- name: Deploy Passit docker_compose: - project_name: "passit" - pull: "yes" - definition: - version: "3.6" - services: - passit_db: - image: "postgres:{{ services.passit.postgres_version }}" - restart: "always" - networks: - - "passit" - volumes: - - "{{ services.passit.volume_folder }}/data:/var/lib/postgresql/data" - environment: - POSTGRES_USER: "passit" - POSTGRES_PASSWORD: "{{ postgres_passwords.passit }}" - - passit_app: - image: "passit/passit:{{ services.passit.version }}" - command: "bin/start.sh" - restart: "always" - networks: - - "passit" - - "postfix" - - "external_services" - environment: - DATABASE_URL: "postgres://passit:{{ postgres_passwords.passit }}@passit_db:5432/passit" - SECRET_KEY: "{{ passit_secret_key }}" - IS_DEBUG: 'False' - EMAIL_URL: "smtp://noop@{{ smtp_host }}:{{ smtp_port }}" - DEFAULT_FROM_EMAIL: "noreply@{{ services.passit.domain }}" - EMAIL_CONFIRMATION_HOST: "https://{{ services.passit.domain }}" - FIDO_SERVER_ID: "{{ services.passit.domain }}" - VIRTUAL_HOST: "{{ services.passit.domain }}" - LETSENCRYPT_HOST: "{{ services.passit.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - - networks: - passit: - postfix: - external: true - external_services: - external: true + project_src: "{{ services.passit.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/rallly.yml b/roles/docker/tasks/services/rallly.yml index 400073f9..e5f2b27b 100644 --- a/roles/docker/tasks/services/rallly.yml +++ b/roles/docker/tasks/services/rallly.yml @@ -8,54 +8,15 @@ - name: Copy Rallly environment file template: src: rallly/env.j2 - dest: "{{ services.rallly.volume_folder }}/env_file" + dest: "{{ services.rallly.volume_folder }}/rallly.env" -- name: Set up Rallly +- name: Upload Compose file for Rallly + template: + src: compose-files/rallly.yml.j2 + dest: "{{ services.rallly.volume_folder }}/docker-compose.yml" + +- name: Deploy Rallly docker_compose: - project_name: "rallly" - pull: "yes" - definition: - version: "3.8" - services: - rallly_db: - image: "postgres:{{ services.rallly.postgres_version }}" - restart: "always" - shm_size: "256mb" - networks: - rallly_internal: - volumes: - - "{{ services.rallly.volume_folder }}/postgres:/var/lib/postgresql/data" - environment: - POSTGRES_PASSWORD: "{{ postgres_passwords.rallly }}" - POSTGRES_DB: "rallly_db" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 5s - timeout: 5s - retries: 5 - - rallly: - image: "lukevella/rallly:{{ services.rallly.version }}" - restart: "always" - networks: - rallly_internal: - external_services: - postfix: - depends_on: - rallly_db: - condition: "service_healthy" - env_file: - - "{{ services.rallly.volume_folder }}/env_file" - environment: - VIRTUAL_HOST: "{{ services.rallly.domain }}" - VIRTUAL_PORT: "3000" - LETSENCRYPT_HOST: "{{ services.rallly.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - - networks: - rallly_internal: - internal: true - external_services: - external: true - postfix: - external: true + project_src: "{{ services.rallly.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/restic.yml b/roles/docker/tasks/services/restic.yml new file mode 100644 index 00000000..c838e265 --- /dev/null +++ b/roles/docker/tasks/services/restic.yml @@ -0,0 +1,59 @@ +# vim: ft=yaml.ansible +--- +- name: Create SSH directory + file: + path: "{{ services.restic.volume_folder }}/ssh" + owner: root + group: root + mode: '0755' + state: directory + +- name: Copy private SSH key + copy: + dest: "{{ services.restic.volume_folder }}/ssh/id_ed25519" + owner: root + group: root + mode: '0600' + content: "{{ restic_secrets.ssh_privkey }}" + +- name: Derive public SSH key + shell: >- + ssh-keygen -f {{ services.restic.volume_folder }}/ssh/id_ed25519 -y + > {{ services.restic.volume_folder }}/ssh/id_ed25519.pub + args: + creates: "{{ services.restic.volume_folder }}/ssh/id_ed25519.pub" + +- name: Set file permissions on public SSH key + file: + path: "{{ services.restic.volume_folder }}/ssh/id_ed25519.pub" + owner: root + group: root + mode: '0644' + state: touch + +- name: Create SSH config + template: + src: restic/ssh.config.j2 + dest: "{{ services.restic.volume_folder }}/ssh/config" + owner: root + group: root + mode: '0600' + +- name: Create SSH known_hosts file + template: + src: restic/ssh.known_hosts.j2 + dest: "{{ services.restic.volume_folder }}/ssh/known_hosts" + owner: root + group: root + mode: '0600' + +- name: Upload Compose file for Restic + template: + src: compose-files/restic.yml.j2 + dest: "{{ services.restic.volume_folder }}/docker-compose.yml" + +- name: Deploy Restic + docker_compose: + project_src: "{{ services.restic.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/restic_backup.yml b/roles/docker/tasks/services/restic_backup.yml deleted file mode 100644 index df0c278d..00000000 --- a/roles/docker/tasks/services/restic_backup.yml +++ /dev/null @@ -1,89 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create SSH directory - file: - path: "{{ services.restic.volume_folder }}/ssh" - owner: root - group: root - mode: '0755' - state: directory - -- name: Copy private SSH key - copy: - dest: "{{ services.restic.volume_folder }}/ssh/id_ed25519" - owner: root - group: root - mode: '0600' - content: "{{ restic_secrets.ssh_privkey }}" - -- name: Derive public SSH key - shell: >- - ssh-keygen -f {{ services.restic.volume_folder }}/ssh/id_ed25519 -y - > {{ services.restic.volume_folder }}/ssh/id_ed25519.pub - args: - creates: "{{ services.restic.volume_folder }}/ssh/id_ed25519.pub" - -- name: Set file permissions on public SSH key - file: - path: "{{ services.restic.volume_folder }}/ssh/id_ed25519.pub" - owner: root - group: root - mode: '0644' - state: touch - -- name: Create SSH config - template: - src: restic/ssh.config.j2 - dest: "{{ services.restic.volume_folder }}/ssh/config" - owner: root - group: root - mode: '0600' - -- name: Create SSH known_hosts file - template: - src: restic/ssh.known_hosts.j2 - dest: "{{ services.restic.volume_folder }}/ssh/known_hosts" - owner: root - group: root - mode: '0600' - -- name: Setup restic backup - docker_compose: - project_name: restic - pull: true - definition: - version: '3.6' - services: - backup: - image: mazzolino/restic:{{ services.restic.version }} - restart: always - environment: - RUN_ON_STARTUP: "false" - BACKUP_CRON: "0 30 3 * * *" - RESTIC_REPOSITORY: "sftp:{{ services.restic.user }}@{{ services.restic.domain }}:{{ services.restic.repository }}" - RESTIC_PASSWORD: "{{ restic_secrets.repository_password }}" - RESTIC_BACKUP_SOURCES: "/mnt/volumes" - RESTIC_BACKUP_ARGS: >- - --tag datacoop-volumes - --exclude '*.tmp' - --verbose - RESTIC_FORGET_ARGS: >- - --keep-last 10 - --keep-daily 7 - --keep-weekly 5 - --keep-monthly 12 - TZ: Europe/Copenhagen - volumes: - - "{{ services.restic.volume_folder }}/ssh:/run/secrets/.ssh:ro" - - /docker-volumes:/mnt/volumes:ro - - prune: - image: "mazzolino/restic:{{ services.restic.version }}" - environment: - RUN_ON_STARTUP: "false" - PRUNE_CRON: "0 0 4 * * *" - RESTIC_REPOSITORY: "sftp:{{ services.restic.user }}@{{ services.restic.domain }}:{{ services.restic.repository }}" - RESTIC_PASSWORD: "{{ restic_secrets.repository_password }}" - TZ: Europe/copenhagen - volumes: - - "{{ services.restic.volume_folder }}/ssh:/run/secrets/.ssh:ro" diff --git a/roles/docker/templates/compose-files/drone.yml.j2 b/roles/docker/templates/compose-files/drone.yml.j2 new file mode 100644 index 00000000..d62eb4b1 --- /dev/null +++ b/roles/docker/templates/compose-files/drone.yml.j2 @@ -0,0 +1,40 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + drone: + image: drone/drone:{{ services.drone.version }} + restart: unless-stopped + networks: + - default + - external_services + volumes: + - ".:/data" + - "/var/run/docker.sock:/var/run/docker.sock" + environment: + DRONE_GITEA_SERVER: https://{{ services.forgejo.domain }} + DRONE_GITEA_CLIENT_ID: "{{ drone_secrets.oauth_client_id }}" + DRONE_GITEA_CLIENT_SECRET: "{{ drone_secrets.oauth_client_secret }}" + DRONE_GIT_ALWAYS_AUTH: true + DRONE_SERVER_HOST: "{{ services.drone.domain }}" + DRONE_SERVER_PROTO: https + DRONE_RPC_SECRET: "{{ drone_secrets.rpc_shared_secret }}" + VIRTUAL_HOST: "{{ services.drone.domain }}" + LETSENCRYPT_HOST: "{{ services.drone.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + + runner: + image: drone/drone-runner-docker:{{ services.drone.version }} + restart: unless-stopped + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + environment: + DRONE_RPC_HOST: "{{ services.drone.domain }}" + DRONE_RPC_PROTO: https + DRONE_RPC_SECRET: "{{ drone_secrets.rpc_shared_secret }}" + DRONE_RUNNER_CAPACITY: 2 + DRONE_RUNNER_NAME: data.coop_drone_runner + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/hedgedoc.yml.j2 b/roles/docker/templates/compose-files/hedgedoc.yml.j2 new file mode 100644 index 00000000..b361116c --- /dev/null +++ b/roles/docker/templates/compose-files/hedgedoc.yml.j2 @@ -0,0 +1,44 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + db: + image: postgres:{{ services.hedgedoc.postgres_version }} + restart: unless-stopped + volumes: + - "./db:/var/lib/postgresql/data" + environment: + POSTGRES_USER: codimd + POSTGRES_PASSWORD: "{{ postgres_passwords.hedgedoc }}" + POSTGRES_DB: codimd + + app: + image: quay.io/hedgedoc/hedgedoc:{{ services.hedgedoc.version }} + volumes: + - "./hedgedoc/uploads:/hedgedoc/public/uploads" + - "./sso.data.coop.pem:/sso.data.coop.pem" + restart: unless-stopped + networks: + - default + - external_services + environment: + CMD_DB_URL: postgres://codimd:{{ postgres_passwords.hedgedoc }}@db:5432/codimd + CMD_DOMAIN: "{{ services.hedgedoc.domain }}" + CMD_ALLOW_EMAIL_REGISTER: False + CMD_IMAGE_UPLOAD_TYPE: filesystem + CMD_EMAIL: False + CMD_SAML_IDPCERT: /sso.data.coop.pem + CMD_SAML_IDPSSOURL: https://{{ services.keycloak.domain }}/auth/realms/datacoop/protocol/saml + CMD_SAML_ISSUER: hedgedoc + CMD_SAML_IDENTIFIERFORMAT: urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified + CMD_USECDN: false + CMD_PROTOCOL_USESSL: true + VIRTUAL_HOST: "{{ services.hedgedoc.domain }}" + LETSENCRYPT_HOST: "{{ services.hedgedoc.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + depends_on: + - db + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/keycloak.yml.j2 b/roles/docker/templates/compose-files/keycloak.yml.j2 new file mode 100644 index 00000000..43c5bb9f --- /dev/null +++ b/roles/docker/templates/compose-files/keycloak.yml.j2 @@ -0,0 +1,42 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + db: + image: postgres:{{ services.keycloak.postgres_version }} + restart: unless-stopped + volumes: + - "./data:/var/lib/postgresql/data" + environment: + POSTGRES_USER: keycloak + POSTGRES_PASSWORD: "{{ postgres_passwords.keycloak }}" + POSTGRES_DB: keycloak + + app: + image: quay.io/keycloak/keycloak:{{ services.keycloak.version }} + restart: unless-stopped + networks: + - default + - postfix + - external_services + command: + - "start" + - "--db=postgres" + - "--db-url=jdbc:postgresql://db:5432/keycloak" + - "--db-username=keycloak" + - "--db-password={{ postgres_passwords.keycloak }}" + - "--hostname={{ services.keycloak.domain }}" + - "--proxy=edge" + - "--https-port=8080" + - "--http-relative-path=/auth" + environment: + VIRTUAL_HOST: "{{ services.keycloak.domain }}" + VIRTUAL_PORT: "8080" + LETSENCRYPT_HOST: "{{ services.keycloak.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + postfix: + external: true + external_services: + external: true diff --git a/roles/docker/templates/compose-files/mailu.yml.j2 b/roles/docker/templates/compose-files/mailu.yml.j2 new file mode 100644 index 00000000..eddc18a9 --- /dev/null +++ b/roles/docker/templates/compose-files/mailu.yml.j2 @@ -0,0 +1,131 @@ +# vim: ft=yaml.docker-compose +version: '3.6' + +services: + postgres: + image: postgres:14-alpine + restart: always + environment: + POSTGRES_DB: mailu + POSTGRES_USER: mailu + POSTGRES_PASSWORD: "{{ postgres_passwords.mailu }}" + volumes: + - "./postgres:/var/lib/postgresql/data" + dns: + - "{{ services.mailu.dns }}" + + redis: + image: redis:alpine + restart: always + volumes: + - "./redis:/data" + depends_on: + - resolver + dns: + - "{{ services.mailu.dns }}" + + front: + image: ghcr.io/mailu/nginx:{{ services.mailu.version }} + restart: always + env_file: mailu.env + environment: + VIRTUAL_HOST: "{{ services.mailu.domain }}" + LETSENCRYPT_HOST: "{{ services.mailu.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + volumes: + - "./certs:/certs" + - "./overrides/nginx:/overrides:ro" + expose: + - "80" + ports: + - "993:993" + - "25:25" + - "587:587" + - "465:465" + networks: + - default + - external_services + + resolver: + image: ghcr.io/mailu/unbound:{{ services.mailu.version }} + restart: always + env_file: mailu.env + networks: + default: + ipv4_address: "{{ services.mailu.dns }}" + + admin: + image: ghcr.io/mailu/admin:{{ services.mailu.version }} + restart: always + env_file: "{{ services.mailu.volume_folder }}/mailu.env" + volumes: + - "./data:/data" + - "./dkim:/dkim" + depends_on: + - redis + - resolver + dns: + - "{{ services.mailu.dns }}" + + imap: + image: ghcr.io/mailu/dovecot:{{ services.mailu.version }} + restart: always + env_file: mailu.env + volumes: + - "./mail:/mail" + - "./overrides/dovecot:/overrides:ro" + depends_on: + - front + - resolver + dns: + - "{{ services.mailu.dns }}" + + smtp: + image: ghcr.io/mailu/postfix:{{ services.mailu.version }} + restart: always + env_file: mailu.env + volumes: + - "./mailqueue:/queue" + - "./overrides/postfix:/overrides:ro" + depends_on: + - front + - resolver + dns: + - "{{ services.mailu.dns }}" + + antispam: + image: ghcr.io/mailu/rspamd:{{ services.mailu.version }} + hostname: antispam + restart: always + env_file: mailu.env + volumes: + - "./filter:/var/lib/rspamd" + - "./overrides/rspamd:/etc/rspamd/override.d:ro" + depends_on: + - front + - resolver + dns: + - "{{ services.mailu.dns }}" + + webmail: + image: ghcr.io/mailu/rainloop:{{ services.mailu.version }} + restart: always + env_file: mailu.env + volumes: + - "./webmail:/data" + - "./overrides/rainloop:/overrides:ro" + depends_on: + - imap + - resolver + dns: + - "{{ services.mailu.dns }}" + +networks: + default: + driver: bridge + ipam: + driver: default + config: + - subnet: "{{ services.mailu.subnet }}" + external_services: + external: true diff --git a/roles/docker/templates/compose-files/mastodon.yml.j2 b/roles/docker/templates/compose-files/mastodon.yml.j2 new file mode 100644 index 00000000..2fcec433 --- /dev/null +++ b/roles/docker/templates/compose-files/mastodon.yml.j2 @@ -0,0 +1,146 @@ +# vim: ft=yaml.docker-compose +x-sidekiq: &sidekiq + image: tootsuite/mastodon:{{ services.mastodon.version }} + restart: always + env_file: mastodon.env + networks: + - default + - postfix + - external_services + volumes: + - "./mastodon_data:/mastodon/public/system" + healthcheck: + test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"] + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + +version: "3.8" + +services: + db: + restart: always + image: postgres:{{ services.mastodon.postgres_version }} + shm_size: 256mb + volumes: + - "./postgres_data:/var/lib/postgresql/data" + - "./postgres_config:/config:ro" + command: postgres -c config_file=/config/postgresql.conf + environment: + POSTGRES_HOST_AUTH_METHOD: trust + healthcheck: + test: ['CMD', 'pg_isready', '-U', 'postgres'] + + redis: + restart: always + image: redis:{{ services.mastodon.redis_version }} + volumes: + - "./redis_data:/data" + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + + web: + image: tootsuite/mastodon:{{ services.mastodon.version }} + restart: always + env_file: mastodon.env + command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" + networks: + - default + - external_services + volumes: + - "./mastodon_data:/mastodon/public/system" + environment: + MAX_THREADS: 10 + WEB_CONCURRENCY: 3 + VIRTUAL_HOST: "{{ services.mastodon.domain }}" + VIRTUAL_PORT: "3000" + VIRTUAL_PATH: / + LETSENCRYPT_HOST: "{{ services.mastodon.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + healthcheck: + test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:3000/health || exit 1'] + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + + streaming: + image: tootsuite/mastodon:{{ services.mastodon.version }} + restart: always + env_file: mastodon.env + command: node ./streaming + networks: + - default + - external_services + ports: + - "127.0.0.1:4000:4000" + environment: + DB_POOL: 15 + VIRTUAL_HOST: "{{ services.mastodon.domain }}" + VIRTUAL_PORT: "4000" + VIRTUAL_PATH: "/api/v1/streaming" + healthcheck: + test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:4000/api/v1/streaming/health || exit 1'] + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + + # sidekiq-default-push-pull: DB_POOL = 25, -c 25 for 25 connections + sidekiq-default-push-pull: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q default -q push -q pull + environment: + DB_POOL: 25 + + # sidekiq-default-pull-push: DB_POOL = 25, -c 25 for 25 connections + sidekiq-default-pull-push: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q default -q pull -q push + environment: + DB_POOL: 25 + + # sidekiq-pull-default-push: DB_POOL = 25, -c 25 for 25 connections + sidekiq-pull-default-push: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q pull -q default -q push + environment: + DB_POOL: 25 + + # sidekiq-push-default-pull: DB_POOL = 25, -c 25 for 25 connections + sidekiq-push-default-pull: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q push -q default -q pull + environment: + DB_POOL: 25 + + # sidekiq-push-scheduler: DB_POOL = 5, -c 5 for 5 connections + sidekiq-push-scheduler: + <<: *sidekiq + command: bundle exec sidekiq -c 5 -q push -q scheduler + environment: + DB_POOL: 5 + + # sidekiq-push-mailers: DB_POOL = 5, -c 5 for 5 connections + sidekiq-push-mailers: + <<: *sidekiq + command: bundle exec sidekiq -c 5 -q push -q mailers + environment: + DB_POOL: 5 + + # sidekiq-push-ingress: DB_POOL = 10, -c 10 for 10 connections + sidekiq-push-ingress: + <<: *sidekiq + command: bundle exec sidekiq -c 10 -q push -q ingress + environment: + DB_POOL: 10 + +networks: + external_services: + external: true + postfix: + external: true diff --git a/roles/docker/templates/compose-files/matrix_element.yml.j2 b/roles/docker/templates/compose-files/matrix_element.yml.j2 new file mode 100644 index 00000000..f787e61f --- /dev/null +++ b/roles/docker/templates/compose-files/matrix_element.yml.j2 @@ -0,0 +1,52 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + postgres: + image: postgres:{{ services.matrix.postgres_version }} + restart: unless-stopped + volumes: + - "./db:/var/lib/postgresql/data" + environment: + POSTGRES_USER: synapse + POSTGRES_PASSWORD: "{{ postgres_passwords.matrix }}" + + synapse: + image: matrixdotorg/synapse:{{ services.matrix.version }} + restart: unless-stopped + networks: + - default + - external_services + - postfix + volumes: + - "./data:/data" + environment: + SYNAPSE_CONFIG_PATH: /data/homeserver.yaml + SYNAPSE_CACHE_FACTOR: "2" + SYNAPSE_LOG_LEVEL: INFO + VIRTUAL_HOST: "{{ services.matrix.domain }}" + VIRTUAL_PORT: "8008" + LETSENCRYPT_HOST: "{{ services.matrix.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + + element: + image: avhost/docker-matrix-element:{{ services.element.version }} + restart: unless-stopped + networks: + - default + - external_services + expose: + - "8080" + volumes: + - "{{ services.element.volume_folder }}/data:/data" + environment: + VIRTUAL_HOST: "{{ services.element.domains | join(',') }}" + VIRTUAL_PORT: "8080" + LETSENCRYPT_HOST: "{{ services.element.domains | join(',') }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + external_services: + external: true + postfix: + external: true diff --git a/roles/docker/templates/compose-files/membersystem.yml.j2 b/roles/docker/templates/compose-files/membersystem.yml.j2 new file mode 100644 index 00000000..fae37672 --- /dev/null +++ b/roles/docker/templates/compose-files/membersystem.yml.j2 @@ -0,0 +1,44 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + app: + image: docker.data.coop/membersystem:{{ services.membersystem.version }} + restart: always + user: "$UID:$GID" + tty: true + networks: + - default + - external_services + - postfix + environment: + SECRET_KEY: "{{ membersystem_secrets.secret_key }}" + DATABASE_URL: postgres://postgres:{{ postgres_passwords.membersystem }}@postgres:5432/postgres + POSTGRES_HOST: postgres + POSTGRES_PORT: 5432 + EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend + EMAIL_URL: smtp://noop@{{ smtp_host }}:{{ smtp_port }} + VIRTUAL_HOST: "{{ services.membersystem.domain }}" + VIRTUAL_PORT: "8000" + LETSENCRYPT_HOST: "{{ services.membersystem.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + ALLOWED_HOSTS: "{{ services.membersystem.domain }}" + CSRF_TRUSTED_ORIGINS: https://{{ services.membersystem.domain }} + DJANGO_ADMINS: "{{ services.membersystem.django_admins }}" + DEFAULT_FROM_EMAIL: noreply@{{ services.membersystem.domain }} + depends_on: + - postgres + + postgres: + image: postgres:{{ services.membersystem.postgres_version }} + restart: always + volumes: + - "./postgres/data:/var/lib/postgresql/data" + environment: + POSTGRES_PASSWORD: "{{ postgres_passwords.membersystem }}" + +networks: + external_services: + external: true + postfix: + external: true diff --git a/roles/docker/templates/compose-files/nextcloud.yml.j2 b/roles/docker/templates/compose-files/nextcloud.yml.j2 new file mode 100644 index 00000000..b95c55dd --- /dev/null +++ b/roles/docker/templates/compose-files/nextcloud.yml.j2 @@ -0,0 +1,59 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + postgres: + image: postgres:{{ services.nextcloud.postgres_version }} + restart: unless-stopped + volumes: + - "./postgres:/var/lib/postgresql/data" + environment: + POSTGRES_DB: nextcloud + POSTGRES_PASSWORD: "{{ postgres_passwords.nextcloud }}" + POSTGRES_USER: nextcloud + + redis: + image: redis:{{ services.nextcloud.redis_version }} + restart: unless-stopped + command: redis-server --requirepass {{ nextcloud_secrets.redis_password }} + tmpfs: + - /var/lib/redis + + cron: + image: nextcloud:{{ services.nextcloud.version }} + restart: unless-stopped + entrypoint: /cron.sh + volumes: + - "./app:/var/www/html" + depends_on: + - postgres + - redis + + app: + image: nextcloud:{{ services.nextcloud.version }} + restart: unless-stopped + networks: + - default + - postfix + - external_services + volumes: + - "./app:/var/www/html" + environment: + VIRTUAL_HOST: "{{ services.nextcloud.domain }}" + LETSENCRYPT_HOST: "{{ services.nextcloud.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + POSTGRES_HOST: postgres + POSTGRES_DB: nextcloud + POSTGRES_USER: nextcloud + POSTGRES_PASSWORD: "{{ postgres_passwords.nextcloud }}" + REDIS_HOST: redis + REDIS_HOST_PASSWORD: "{{ nextcloud_secrets.redis_password }}" + depends_on: + - postgres + - redis + +networks: + postfix: + external: true + external_services: + external: true diff --git a/roles/docker/templates/compose-files/passit.yml.j2 b/roles/docker/templates/compose-files/passit.yml.j2 new file mode 100644 index 00000000..810d76e3 --- /dev/null +++ b/roles/docker/templates/compose-files/passit.yml.j2 @@ -0,0 +1,38 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + db: + image: postgres:{{ services.passit.postgres_version }} + restart: always + volumes: + - "./data:/var/lib/postgresql/data" + environment: + POSTGRES_USER: passit + POSTGRES_PASSWORD: "{{ postgres_passwords.passit }}" + + app: + image: passit/passit:{{ services.passit.version }} + command: bin/start.sh + restart: always + networks: + - default + - postfix + - external_services + environment: + DATABASE_URL: postgres://passit:{{ postgres_passwords.passit }}@db:5432/passit + SECRET_KEY: "{{ passit_secret_key }}" + IS_DEBUG: "False" + EMAIL_URL: smtp://noop@{{ smtp_host }}:{{ smtp_port }} + DEFAULT_FROM_EMAIL: noreply@{{ services.passit.domain }} + EMAIL_CONFIRMATION_HOST: https://{{ services.passit.domain }} + FIDO_SERVER_ID: "{{ services.passit.domain }}" + VIRTUAL_HOST: "{{ services.passit.domain }}" + LETSENCRYPT_HOST: "{{ services.passit.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + postfix: + external: true + external_services: + external: true diff --git a/roles/docker/templates/compose-files/rallly.yml.j2 b/roles/docker/templates/compose-files/rallly.yml.j2 new file mode 100644 index 00000000..f8cf987c --- /dev/null +++ b/roles/docker/templates/compose-files/rallly.yml.j2 @@ -0,0 +1,41 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + db: + image: postgres:{{ services.rallly.postgres_version }} + restart: always + shm_size: 256mb + volumes: + - "./postgres:/var/lib/postgresql/data" + environment: + POSTGRES_PASSWORD: "{{ postgres_passwords.rallly }}" + POSTGRES_DB: rallly_db + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 5s + retries: 5 + + rallly: + image: lukevella/rallly:{{ services.rallly.version }} + restart: always + networks: + - default + - external_services + - postfix + depends_on: + db: + condition: service_healthy + env_file: rallly.env + environment: + VIRTUAL_HOST: "{{ services.rallly.domain }}" + VIRTUAL_PORT: "3000" + LETSENCRYPT_HOST: "{{ services.rallly.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + external_services: + external: true + postfix: + external: true diff --git a/roles/docker/templates/compose-files/restic.yml.j2 b/roles/docker/templates/compose-files/restic.yml.j2 new file mode 100644 index 00000000..1f2ed2b0 --- /dev/null +++ b/roles/docker/templates/compose-files/restic.yml.j2 @@ -0,0 +1,37 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + backup: + image: mazzolino/restic:{{ services.restic.version }} + restart: always + environment: + RUN_ON_STARTUP: false + BACKUP_CRON: "0 30 3 * * *" + RESTIC_REPOSITORY: sftp:{{ services.restic.user }}@{{ services.restic.domain }}:{{ services.restic.repository }} + RESTIC_PASSWORD: "{{ restic_secrets.repository_password }}" + RESTIC_BACKUP_SOURCES: /mnt/volumes + RESTIC_BACKUP_ARGS: >- + --tag datacoop-volumes + --exclude '*.tmp' + --verbose + RESTIC_FORGET_ARGS: >- + --keep-last 10 + --keep-daily 7 + --keep-weekly 5 + --keep-monthly 12 + TZ: Europe/Copenhagen + volumes: + - "./ssh:/run/secrets/.ssh:ro" + - "/docker-volumes:/mnt/volumes:ro" + + prune: + image: mazzolino/restic:{{ services.restic.version }} + environment: + RUN_ON_STARTUP: false + PRUNE_CRON: "0 30 4 * * *" + RESTIC_REPOSITORY: sftp:{{ services.restic.user }}@{{ services.restic.domain }}:{{ services.restic.repository }} + RESTIC_PASSWORD: "{{ restic_secrets.repository_password }}" + TZ: Europe/copenhagen + volumes: + - "./ssh:/run/secrets/.ssh:ro" diff --git a/roles/docker/templates/rallly/env.j2 b/roles/docker/templates/rallly/env.j2 index 64036967..5fa89ea7 100644 --- a/roles/docker/templates/rallly/env.j2 +++ b/roles/docker/templates/rallly/env.j2 @@ -1,5 +1,5 @@ NEXT_PUBLIC_BASE_URL="https://{{ services.rallly.domain }}" -DATABASE_URL="postgres://postgres:{{ postgres_passwords.rallly }}@rallly_db:5432/rallly_db" +DATABASE_URL="postgres://postgres:{{ postgres_passwords.rallly }}@db:5432/rallly_db" SECRET_PASSWORD="{{ rallly_secrets.secret_password }}" SUPPORT_EMAIL="noreply@{{ services.rallly.domain }}" SMTP_HOST="{{ smtp_host }}" From 85aa718480224c61ddead116dadc82627dd9d2be Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 30 Sep 2023 16:42:16 +0200 Subject: [PATCH 02/23] Split Matrix and Element into their own Compose stacks --- roles/docker/defaults/main.yml | 7 ++-- roles/docker/tasks/services/element.yml | 32 +++++++++++++++++++ .../{matrix_element.yml => matrix.yml} | 31 ++---------------- .../templates/compose-files/element.yml.j2 | 22 +++++++++++++ .../{matrix_element.yml.j2 => matrix.yml.j2} | 16 ---------- 5 files changed, 60 insertions(+), 48 deletions(-) create mode 100644 roles/docker/tasks/services/element.yml rename roles/docker/tasks/services/{matrix_element.yml => matrix.yml} (56%) create mode 100644 roles/docker/templates/compose-files/element.yml.j2 rename roles/docker/templates/compose-files/{matrix_element.yml.j2 => matrix.yml.j2} (64%) diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 5477414c..46cca1b4 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -92,7 +92,7 @@ services: allowed_sender_domain: true matrix: - file: matrix_element.yml + file: matrix.yml domain: "matrix.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/matrix" version: v1.90.0 @@ -100,9 +100,8 @@ services: allowed_sender_domain: true element: - domains: - - "riot.{{ base_domain }}" - - "element.{{ base_domain }}" + file: element.yml + domain: "element.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/element" version: v1.11.43 diff --git a/roles/docker/tasks/services/element.yml b/roles/docker/tasks/services/element.yml new file mode 100644 index 00000000..b325bdf2 --- /dev/null +++ b/roles/docker/tasks/services/element.yml @@ -0,0 +1,32 @@ +# vim: ft=yaml.ansible +--- +- name: Create Element volume folder + file: + name: "{{ services.element.volume_folder }}/data" + state: directory + +- name: Upload Element config.json + template: + src: element/config.json.j2 + dest: "{{ services.element.volume_folder }}/data/config.json" + +- name: Upload Element riot.im.conf + copy: + src: element/riot.im.conf + dest: "{{ services.element.volume_folder }}/data/riot.im.conf" + +- name: Upload vhost config for Element domain + copy: + src: vhost/element + dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.element.domain }}" + +- name: Upload Compose file for Element + template: + src: compose-files/element.yml.j2 + dest: "{{ services.element.volume_folder }}/docker-compose.yml" + +- name: Deploy Element + docker_compose: + project_src: "{{ services.element.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/matrix_element.yml b/roles/docker/tasks/services/matrix.yml similarity index 56% rename from roles/docker/tasks/services/matrix_element.yml rename to roles/docker/tasks/services/matrix.yml index 605084b3..2a4a6f8d 100644 --- a/roles/docker/tasks/services/matrix_element.yml +++ b/roles/docker/tasks/services/matrix.yml @@ -16,38 +16,13 @@ - name: Create Matrix DB folder file: name: "{{ services.matrix.volume_folder }}/db" - state: "directory" - -- name: Create Element volume folders - file: - name: "{{ services.element.volume_folder }}/{{ volume }}" state: directory - loop: - - "data" - loop_control: - loop_var: volume - -- name: Upload Element config.json - template: - src: element/config.json.j2 - dest: "{{ services.element.volume_folder }}/data/config.json" - -- name: Upload Element riot.im.conf - copy: - src: element/riot.im.conf - dest: "{{ services.element.volume_folder }}/data/riot.im.conf" - name: upload vhost config for matrix domain copy: src: vhost/matrix dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.matrix.domain }}" -- name: Upload vhost config for Element domain - copy: - src: vhost/element - dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ item }}" - loop: "{{ services.element.domains }}" - - name: Upload homeserver.yaml template: src: matrix/homeserver.yaml.j2 @@ -58,12 +33,12 @@ src: matrix/log.config dest: "{{ services.matrix.volume_folder }}/data/matrix.data.coop.log.config" -- name: Upload Compose file for Matrix and Element +- name: Upload Compose file for Matrix template: - src: compose-files/matrix_element.yml.j2 + src: compose-files/matrix.yml.j2 dest: "{{ services.matrix.volume_folder }}/docker-compose.yml" -- name: Deploy Matrix and Element +- name: Deploy Matrix docker_compose: project_src: "{{ services.matrix.volume_folder }}" pull: true diff --git a/roles/docker/templates/compose-files/element.yml.j2 b/roles/docker/templates/compose-files/element.yml.j2 new file mode 100644 index 00000000..2a875cec --- /dev/null +++ b/roles/docker/templates/compose-files/element.yml.j2 @@ -0,0 +1,22 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + element: + image: avhost/docker-matrix-element:{{ services.element.version }} + restart: unless-stopped + networks: + - external_services + expose: + - "8080" + volumes: + - "./data:/data" + environment: + VIRTUAL_HOST: "{{ services.element.domain }}" + VIRTUAL_PORT: "8080" + LETSENCRYPT_HOST: "{{ services.element.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/matrix_element.yml.j2 b/roles/docker/templates/compose-files/matrix.yml.j2 similarity index 64% rename from roles/docker/templates/compose-files/matrix_element.yml.j2 rename to roles/docker/templates/compose-files/matrix.yml.j2 index f787e61f..0bbffa50 100644 --- a/roles/docker/templates/compose-files/matrix_element.yml.j2 +++ b/roles/docker/templates/compose-files/matrix.yml.j2 @@ -29,22 +29,6 @@ services: LETSENCRYPT_HOST: "{{ services.matrix.domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - element: - image: avhost/docker-matrix-element:{{ services.element.version }} - restart: unless-stopped - networks: - - default - - external_services - expose: - - "8080" - volumes: - - "{{ services.element.volume_folder }}/data:/data" - environment: - VIRTUAL_HOST: "{{ services.element.domains | join(',') }}" - VIRTUAL_PORT: "8080" - LETSENCRYPT_HOST: "{{ services.element.domains | join(',') }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - networks: external_services: external: true From 728455f42a732a0f572ae0c54700bbea26fb467e Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 30 Sep 2023 17:19:10 +0200 Subject: [PATCH 03/23] Convert Netdata to a Compose stack, close #80 --- roles/docker/tasks/services/netdata.yml | 31 ++++++---------- .../templates/compose-files/netdata.yml.j2 | 36 +++++++++++++++++++ 2 files changed, 46 insertions(+), 21 deletions(-) create mode 100644 roles/docker/templates/compose-files/netdata.yml.j2 diff --git a/roles/docker/tasks/services/netdata.yml b/roles/docker/tasks/services/netdata.yml index 7cf01e68..3631c99e 100644 --- a/roles/docker/tasks/services/netdata.yml +++ b/roles/docker/tasks/services/netdata.yml @@ -1,23 +1,12 @@ # vim: ft=yaml.ansible --- -- name: setup netdata docker container for system monitoring - docker_container: - name: netdata - image: netdata/netdata:{{ services.netdata.version }} - restart_policy: unless-stopped - hostname: "hevonen.servers.{{ base_domain }}" - capabilities: - - SYS_PTRACE - security_opts: - - apparmor:unconfined - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - networks: - - name: external_services - env: - VIRTUAL_HOST : "{{ services.netdata.domain }}" - LETSENCRYPT_HOST: "{{ services.netdata.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - PGID: "999" +- name: Upload Compose file for Netdata + template: + src: compose-files/netdata.yml.j2 + dest: "{{ services.netdata.volume_folder }}/docker-compose.yml" + +- name: Deploy Netdata + docker_compose: + project_src: "{{ services.netdata.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/templates/compose-files/netdata.yml.j2 b/roles/docker/templates/compose-files/netdata.yml.j2 new file mode 100644 index 00000000..dcf5eada --- /dev/null +++ b/roles/docker/templates/compose-files/netdata.yml.j2 @@ -0,0 +1,36 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + app: + image: netdata/netdata:{{ services.netdata.version }} + restart: unless-stopped + hostname: hevonen.servers.{{ base_domain }} + volumes: + - "/proc:/host/proc:ro" + - "/sys:/host/sys:ro" + - "/etc/os-release:/host/etc/os-release:ro" + networks: + - default + - external_services + environment: + VIRTUAL_HOST : "{{ services.netdata.domain }}" + LETSENCRYPT_HOST: "{{ services.netdata.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + PGID: "999" + DOCKER_HOST: "socket_proxy:2375" + cap_add: + - SYS_PTRACE + security_opt: + - apparmor:unconfined + + socket_proxy: + image: tecnativa/docker-socket-proxy:latest + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + environment: + CONTAINERS: 1 + +networks: + external_services: + external: true From f50831460c93041cac1bca5f6e75ae3360b28297 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 30 Sep 2023 18:15:27 +0200 Subject: [PATCH 04/23] Convert all services to Compose stacks --- roles/docker/defaults/main.yml | 20 ++--- .../docker/tasks/services/docker_registry.yml | 44 ++++++----- roles/docker/tasks/services/drone.yml | 5 ++ roles/docker/tasks/services/forgejo.yml | 48 ++++-------- roles/docker/tasks/services/keycloak.yml | 5 ++ roles/docker/tasks/services/membersystem.yml | 5 ++ roles/docker/tasks/services/netdata.yml | 5 ++ roles/docker/tasks/services/nextcloud.yml | 10 +++ roles/docker/tasks/services/nginx_proxy.yml | 41 +++------- roles/docker/tasks/services/openldap.yml | 77 ++++--------------- roles/docker/tasks/services/portainer.yml | 25 +++--- roles/docker/tasks/services/postfix.yml | 26 +++---- roles/docker/tasks/services/privatebin.yml | 24 +++--- roles/docker/tasks/services/watchtower.yml | 27 ++++--- .../compose-files/docker_registry.yml.j2 | 23 ++++++ .../templates/compose-files/drone.yml.j2 | 2 +- .../templates/compose-files/element.yml.j2 | 2 +- .../templates/compose-files/forgejo.yml.j2 | 37 +++++++++ .../compose-files/nginx_proxy.yml.j2 | 38 +++++++++ .../templates/compose-files/openldap.yml.j2 | 58 ++++++++++++++ .../templates/compose-files/portainer.yml.j2 | 21 +++++ .../templates/compose-files/postfix.yml.j2 | 20 +++++ .../templates/compose-files/privatebin.yml.j2 | 20 +++++ .../templates/compose-files/rallly.yml.j2 | 2 +- .../templates/compose-files/watchtower.yml.j2 | 12 +++ 25 files changed, 374 insertions(+), 223 deletions(-) create mode 100644 roles/docker/templates/compose-files/docker_registry.yml.j2 create mode 100644 roles/docker/templates/compose-files/forgejo.yml.j2 create mode 100644 roles/docker/templates/compose-files/nginx_proxy.yml.j2 create mode 100644 roles/docker/templates/compose-files/openldap.yml.j2 create mode 100644 roles/docker/templates/compose-files/portainer.yml.j2 create mode 100644 roles/docker/templates/compose-files/postfix.yml.j2 create mode 100644 roles/docker/templates/compose-files/privatebin.yml.j2 create mode 100644 roles/docker/templates/compose-files/watchtower.yml.j2 diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 46cca1b4..e0365ce6 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -13,24 +13,21 @@ services: nginx_proxy: file: nginx_proxy.yml - version: "1.3-alpine" volume_folder: "{{ volume_root_folder }}/nginx" - - nginx_acme_companion: - version: "2.2" + version: "1.3-alpine" + acme_companion_version: "2.2" openldap: file: openldap.yml domain: "ldap.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/openldap" version: "1.5.0" - - phpldapadmin: - version: "0.9.0" + phpldapadmin_version: "0.9.0" netdata: file: netdata.yml domain: "netdata.{{ base_domain }}" + volume_folder: "{{ volume_root_folder }}/netdata" version: "v1" portainer: @@ -196,17 +193,12 @@ services: file: membersystem.yml domain: "member.{{ base_domain }}" django_admins: "Vidir:valberg@orn.li" + volume_folder: "{{ volume_root_folder }}/membersystem" version: latest postgres_version: 13-alpine allowed_sender_domain: true - byro: - file: byro.yml - domain: "byro.{{ base_domain }}" - postgres_version: 14-alpine - volume_folder: "{{ volume_root_folder }}/byro-data" - allowed_sender_domain: true - watchtower: file: watchtower.yml + volume_folder: "{{ volume_root_folder }}/watchtower" version: "1.5.3" diff --git a/roles/docker/tasks/services/docker_registry.yml b/roles/docker/tasks/services/docker_registry.yml index 3adee6db..3ef95429 100644 --- a/roles/docker/tasks/services/docker_registry.yml +++ b/roles/docker/tasks/services/docker_registry.yml @@ -1,32 +1,36 @@ # vim: ft=yaml.ansible --- -- name: copy docker registry vhost configuration +- name: Create Docker registry volume folders + file: + path: "{{ services.docker_registry.volume_folder }}/{{ volume }}" + state: directory + loop: + - auth + - registry + loop_control: + loop_var: volume + +- name: Copy docker registry vhost configuration copy: src: vhost/docker_registry dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.docker_registry.domain }}" mode: "0644" -- name: docker registry container - docker_container: - name: registry - image: registry:{{ services.docker_registry.version }} - restart_policy: always - volumes: - - "{{ services.docker_registry.volume_folder }}/registry:/var/lib/registry" - - "{{ services.docker_registry.volume_folder }}/auth:/auth" - networks: - - name: external_services - env: - VIRTUAL_HOST: "{{ services.docker_registry.domain }}" - LETSENCRYPT_HOST: "{{ services.docker_registry.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - REGISTRY_AUTH: "htpasswd" - REGISTRY_AUTH_HTPASSWD_PATH: "/auth/htpasswd" - REGISTRY_AUTH_HTPASSWD_REALM: "data.coop docker registry" +- name: Upload Compose file for Docker registry + template: + src: compose-files/docker_registry.yml.j2 + dest: "{{ services.docker_registry.volume_folder }}/docker-compose.yml" -- name: generate htpasswd file - shell: "docker exec -it registry htpasswd -Bbn docker {{ docker_password }} > {{ services.docker_registry.volume_folder }}/auth/htpasswd" +- name: Deploy Docker registry + docker_compose: + project_src: "{{ services.docker_registry.volume_folder }}" + pull: true + state: present + +- name: Generate htpasswd file + shell: "docker compose exec registry htpasswd -Bbn docker {{ docker_password }} > auth/htpasswd" args: + chdir: "{{ services.docker_registry.volume_folder }}" creates: "{{ services.docker_registry.volume_folder }}/auth/htpasswd" - name: log in to registry diff --git a/roles/docker/tasks/services/drone.yml b/roles/docker/tasks/services/drone.yml index de8720e5..8e4fa257 100644 --- a/roles/docker/tasks/services/drone.yml +++ b/roles/docker/tasks/services/drone.yml @@ -1,5 +1,10 @@ # vim: ft=yaml.ansible --- +- name: Create Drone volume folder + file: + path: "{{ services.drone.volume_folder }}" + state: directory + - name: Upload Compose file for Drone template: src: compose-files/drone.yml.j2 diff --git a/roles/docker/tasks/services/forgejo.yml b/roles/docker/tasks/services/forgejo.yml index 9978b82d..826a190c 100644 --- a/roles/docker/tasks/services/forgejo.yml +++ b/roles/docker/tasks/services/forgejo.yml @@ -1,37 +1,17 @@ # vim: ft=yaml.ansible --- -- name: Create Docker network for Forgejo - docker_network: - name: forgejo +- name: Create Forgejo volume folder + file: + name: "{{ services.portainer.volume_folder }}" + state: directory -# old DNS: 138.68.71.153 -- name: Set up Forgejo container - docker_container: - name: forgejo - image: codeberg.org/forgejo/forgejo:{{ services.forgejo.version }} - restart_policy: unless-stopped - networks: - - name: forgejo - - name: postfix - - name: external_services - volumes: - - "{{ services.forgejo.volume_folder }}:/data" - published_ports: - - "22:22" - env: - VIRTUAL_HOST: "{{ services.forgejo.domain }}" - VIRTUAL_PORT: "3000" - LETSENCRYPT_HOST: "{{ services.forgejo.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - # Forgejo customization, see: https://docs.gitea.io/en-us/install-with-docker/#customization - # https://docs.gitea.io/en-us/config-cheat-sheet/#security-security - FORGEJO__mailer__ENABLED: "true" - FORGEJO__mailer__FROM: "noreply@{{ services.forgejo.domain }}" - FORGEJO__mailer__PROTOCOL: "smtp" - FORGEJO__mailer__SMTP_ADDR: "{{ smtp_host }}:{{ smtp_port }}" - FORGEJO__security__LOGIN_REMEMBER_DAYS: "60" - FORGEJO__security__PASSWORD_COMPLEXITY: "off" - FORGEJO__security__MIN_PASSWORD_LENGTH: "8" - FORGEJO__security__PASSWORD_CHECK_PWN: "true" - FORGEJO__service__ENABLE_NOTIFY_MAIL: "true" - FORGEJO__service__REGISTER_EMAIL_CONFIRM: "true" +- name: Upload Compose file for Forgejo + template: + src: compose-files/forgejo.yml.j2 + dest: "{{ services.forgejo.volume_folder }}/docker-compose.yml" + +- name: Deploy Forgejo + docker_compose: + project_src: "{{ services.forgejo.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/keycloak.yml b/roles/docker/tasks/services/keycloak.yml index 45feb25b..ff341b9a 100644 --- a/roles/docker/tasks/services/keycloak.yml +++ b/roles/docker/tasks/services/keycloak.yml @@ -1,5 +1,10 @@ # vim: ft=yaml.ansible --- +- name: Create Keycloak volume folder + file: + path: "{{ services.keycloak.volume_folder }}/data" + state: directory + - name: Upload Compose file for for Keycloak template: src: compose-files/keycloak.yml.j2 diff --git a/roles/docker/tasks/services/membersystem.yml b/roles/docker/tasks/services/membersystem.yml index ba9135e8..357c1695 100644 --- a/roles/docker/tasks/services/membersystem.yml +++ b/roles/docker/tasks/services/membersystem.yml @@ -1,5 +1,10 @@ # vim: ft=yaml.ansible --- +- name: Create Membersystem volume folder + file: + name: "{{ services.membersystem.volume_folder }}" + state: directory + - name: Upload Compose file for Membersystem template: src: compose-files/membersystem.yml.j2 diff --git a/roles/docker/tasks/services/netdata.yml b/roles/docker/tasks/services/netdata.yml index 3631c99e..e5234b62 100644 --- a/roles/docker/tasks/services/netdata.yml +++ b/roles/docker/tasks/services/netdata.yml @@ -1,5 +1,10 @@ # vim: ft=yaml.ansible --- +- name: Create Netdata volume folder + file: + path: "{{ services.netdata.volume_folder }}" + state: directory + - name: Upload Compose file for Netdata template: src: compose-files/netdata.yml.j2 diff --git a/roles/docker/tasks/services/nextcloud.yml b/roles/docker/tasks/services/nextcloud.yml index acfa587c..7273bcf0 100644 --- a/roles/docker/tasks/services/nextcloud.yml +++ b/roles/docker/tasks/services/nextcloud.yml @@ -1,5 +1,15 @@ # vim: ft=yaml.ansible --- +- name: Create Nextcloud volume folders + file: + path: "{{ services.nextcloud.volume_folder }}/{{ volume }}" + state: directory + loop: + - app + - postgres + loop_control: + loop_var: volume + - name: upload vhost config for cloud.data.coop copy: src: vhost/nextcloud diff --git a/roles/docker/tasks/services/nginx_proxy.yml b/roles/docker/tasks/services/nginx_proxy.yml index 2f92611e..68659529 100644 --- a/roles/docker/tasks/services/nginx_proxy.yml +++ b/roles/docker/tasks/services/nginx_proxy.yml @@ -13,36 +13,13 @@ loop_control: loop_var: volume -- name: nginx proxy container - docker_container: - name: nginx-proxy - image: nginxproxy/nginx-proxy:{{ services.nginx_proxy.version }} - restart_policy: always - networks: - - name: external_services - published_ports: - - "80:80" - - "443:443" - volumes: - - "{{ services.nginx_proxy.volume_folder }}/conf:/etc/nginx/conf.d" - - "{{ services.nginx_proxy.volume_folder }}/vhost:/etc/nginx/vhost.d" - - "{{ services.nginx_proxy.volume_folder }}/html:/usr/share/nginx/html" - - "{{ services.nginx_proxy.volume_folder }}/dhparam:/etc/nginx/dhparam" - - "{{ services.nginx_proxy.volume_folder }}/certs:/etc/nginx/certs:ro" - - /var/run/docker.sock:/tmp/docker.sock:ro - -- name: nginx letsencrypt container - docker_container: - name: nginx-proxy-le - image: nginxproxy/acme-companion:{{ services.nginx_acme_companion.version }} - restart_policy: always - volumes: - - "{{ services.nginx_proxy.volume_folder }}/vhost:/etc/nginx/vhost.d" - - "{{ services.nginx_proxy.volume_folder }}/html:/usr/share/nginx/html" - - "{{ services.nginx_proxy.volume_folder }}/dhparam:/etc/nginx/dhparam:ro" - - "{{ services.nginx_proxy.volume_folder }}/certs:/etc/nginx/certs" - - /var/run/docker.sock:/var/run/docker.sock:ro - env: - NGINX_PROXY_CONTAINER: nginx-proxy - when: letsencrypt_enabled +- name: Upload Compose file for nginx-proxy + template: + src: compose-files/nginx_proxy.yml.j2 + dest: "{{ services.nginx_proxy.volume_folder }}/docker-compose.yml" +- name: Deploy nginx-proxy + docker_compose: + project_src: "{{ services.nginx_proxy.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/openldap.yml b/roles/docker/tasks/services/openldap.yml index 4aace814..b477a5bf 100644 --- a/roles/docker/tasks/services/openldap.yml +++ b/roles/docker/tasks/services/openldap.yml @@ -1,74 +1,23 @@ # vim: ft=yaml.ansible --- -- name: create ldap volume folders +- name: Create OpenLDAP volume folders file: name: "{{ services.openldap.volume_folder }}/{{ volume }}" state: directory loop: - - "var/lib/ldap" - - "etc/slapd" - - "certs" + - var/lib/ldap + - etc/slapd + - certs loop_control: loop_var: volume -- name: Create a network for ldap - docker_network: - name: ldap +- name: Upload Compose file for OpenLDAP + template: + src: compose-files/openldap.yml.j2 + dest: "{{ services.openldap.volume_folder }}/docker-compose.yml" -- name: openLDAP container - docker_container: - name: openldap - image: osixia/openldap:{{ services.openldap.version }} - tty: true - interactive: true - restart_policy: unless-stopped - volumes: - - "{{ services.openldap.volume_folder }}/var/lib/ldap:/var/lib/ldap" - - "{{ services.openldap.volume_folder }}/etc/slapd.d:/etc/ldap/slapd.d" - - "{{ services.openldap.volume_folder }}/certs:/container/service/slapd/assets/certs/" - published_ports: - - "389:389" - - "636:636" - hostname: "{{ services.openldap.domain }}" - domainname: "{{ services.openldap.domain }}" # important: same as hostname - networks: - - name: ldap - env: - LDAP_LOG_LEVEL: "256" - LDAP_ORGANISATION: "{{ base_domain }}" - LDAP_DOMAIN: "{{ base_domain }}" - LDAP_BASE_DN: "" - LDAP_ADMIN_PASSWORD: "{{ ldap_admin_password }}" - LDAP_CONFIG_PASSWORD: "{{ ldap_config_password }}" - LDAP_READONLY_USER: "false" - LDAP_RFC2307BIS_SCHEMA: "false" - LDAP_BACKEND: "mdb" - LDAP_TLS: "true" - LDAP_TLS_CRT_FILENAME: "ldap.crt" - LDAP_TLS_KEY_FILENAME: "ldap.key" - LDAP_TLS_CA_CRT_FILENAME: "ca.crt" - LDAP_TLS_ENFORCE: "false" - LDAP_TLS_CIPHER_SUITE: "SECURE256:-VERS-SSL3.0" - LDAP_TLS_PROTOCOL_MIN: "3.1" - LDAP_TLS_VERIFY_CLIENT: "demand" - LDAP_REPLICATION: "false" - KEEP_EXISTING_CONFIG: "false" - LDAP_REMOVE_CONFIG_AFTER_SETUP: "true" - LDAP_SSL_HELPER_PREFIX: "ldap" - -- name: phpLDAPadmin container - docker_container: - name: phpldapadmin - image: osixia/phpldapadmin:{{ services.phpldapadmin.version }} - restart_policy: unless-stopped - networks: - - name: external_services - - name: ldap - env: - PHPLDAPADMIN_LDAP_HOSTS: "openldap" - PHPLDAPADMIN_HTTPS: "false" - PHPLDAPADMIN_TRUST_PROXY_SSL: "true" - - VIRTUAL_HOST: "{{ services.openldap.domain }}" - LETSENCRYPT_HOST: "{{ services.openldap.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" +- name: Deploy OpenLDAP + docker_compose: + project_src: "{{ services.openldap.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/portainer.yml b/roles/docker/tasks/services/portainer.yml index dae0e871..5f158c91 100644 --- a/roles/docker/tasks/services/portainer.yml +++ b/roles/docker/tasks/services/portainer.yml @@ -5,18 +5,13 @@ name: "{{ services.portainer.volume_folder }}" state: directory -- name: run portainer - docker_container: - name: portainer - image: portainer/portainer-ee:{{ services.portainer.version }} - restart_policy: always - networks: - - name: external_services - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - "{{ services.portainer.volume_folder }}:/data" - env: - VIRTUAL_HOST: "{{ services.portainer.domain }}" - VIRTUAL_PORT: "9000" - LETSENCRYPT_HOST: "{{ services.portainer.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" +- name: Upload Compose file for Portainer + template: + src: compose-files/portainer.yml.j2 + dest: "{{ services.portainer.volume_folder }}/docker-compose.yml" + +- name: Deploy Portainer + docker_compose: + project_src: "{{ services.portainer.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/postfix.yml b/roles/docker/tasks/services/postfix.yml index ece525e3..5298b15d 100644 --- a/roles/docker/tasks/services/postfix.yml +++ b/roles/docker/tasks/services/postfix.yml @@ -1,6 +1,6 @@ # vim: ft=yaml.ansible --- -- name: Set up network for postfix +- name: Set up network for Postfix docker_network: name: postfix ipam_config: @@ -12,17 +12,13 @@ name: "{{ services.postfix.volume_folder }}/dkim" state: directory -- name: Set up Postfix Docker container for outgoing mail from services - docker_container: - name: postfix - image: boky/postfix:{{ services.postfix.version }} - restart_policy: always - networks: - - name: postfix - volumes: - - "{{ services.postfix.volume_folder }}/dkim:/etc/opendkim/keys" - env: - # Get all services which have allowed_sender_domain defined - ALLOWED_SENDER_DOMAINS: "{{ services | dict2items | selectattr('value.allowed_sender_domain', 'true') | map(attribute='value.domain') | join(' ') }}" - HOSTNAME: "{{ services.postfix.domain }}" # the name the smtp server will identify itself as - DKIM_AUTOGENERATE: "true" +- name: Upload Compose file for Postfix + template: + src: compose-files/postfix.yml.j2 + dest: "{{ services.forgejo.volume_folder }}/docker-compose.yml" + +- name: Deploy Postfix + docker_compose: + project_src: "{{ services.postfix.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/privatebin.yml b/roles/docker/tasks/services/privatebin.yml index 354d81cb..09d648ca 100644 --- a/roles/docker/tasks/services/privatebin.yml +++ b/roles/docker/tasks/services/privatebin.yml @@ -15,17 +15,13 @@ src: privatebin/conf.php dest: "{{ services.privatebin.volume_folder }}/cfg/conf.php" -- name: privatebin app container - docker_container: - name: privatebin - image: jgeusebroek/privatebin:{{ services.privatebin.version }} - restart_policy: unless-stopped - volumes: - - "{{ services.privatebin.volume_folder }}/cfg:/privatebin/cfg" - - "{{ services.privatebin.volume_folder }}/data:/privatebin/data" - networks: - - name: external_services - env: - VIRTUAL_HOST: "{{ services.privatebin.domain }}" - LETSENCRYPT_HOST: "{{ services.privatebin.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" +- name: Upload Compose file for PrivateBin + template: + src: compose-files/privatebin.yml.j2 + dest: "{{ services.privatebin.volume_folder }}/docker-compose.yml" + +- name: Deploy PrivateBin + docker_compose: + project_src: "{{ services.private.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/tasks/services/watchtower.yml b/roles/docker/tasks/services/watchtower.yml index c64c7f25..e528024d 100644 --- a/roles/docker/tasks/services/watchtower.yml +++ b/roles/docker/tasks/services/watchtower.yml @@ -1,14 +1,17 @@ # vim: ft=yaml.ansible --- -- name: watchtower container - docker_container: - name: watchtower - image: containrrr/watchtower:{{ services.watchtower.version }} - restart_policy: unless-stopped - networks: - - name: external_services - env: - WATCHTOWER_POLL_INTERVAL: "60" - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - - "/root/.docker/config.json:/config.json:ro" +- name: Create Watchtower volume folder + file: + name: "{{ services.watchtower.volume_folder }}" + state: directory + +- name: Upload Compose file for Watchtower + template: + src: compose-files/watchtower.yml.j2 + dest: "{{ services.watchtower.volume_folder }}/docker-compose.yml" + +- name: Deploy Watchtower + docker_compose: + project_src: "{{ services.watchtower.volume_folder }}" + pull: true + state: present diff --git a/roles/docker/templates/compose-files/docker_registry.yml.j2 b/roles/docker/templates/compose-files/docker_registry.yml.j2 new file mode 100644 index 00000000..1e0d69c3 --- /dev/null +++ b/roles/docker/templates/compose-files/docker_registry.yml.j2 @@ -0,0 +1,23 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + app: + image: registry:{{ services.docker_registry.version }} + restart: always + networks: + - external_services + volumes: + - "./registry:/var/lib/registry" + - "./auth:/auth" + environment: + VIRTUAL_HOST: "{{ services.docker_registry.domain }}" + LETSENCRYPT_HOST: "{{ services.docker_registry.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + REGISTRY_AUTH: "htpasswd" + REGISTRY_AUTH_HTPASSWD_PATH: "/auth/htpasswd" + REGISTRY_AUTH_HTPASSWD_REALM: "data.coop docker registry" + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/drone.yml.j2 b/roles/docker/templates/compose-files/drone.yml.j2 index d62eb4b1..377720a4 100644 --- a/roles/docker/templates/compose-files/drone.yml.j2 +++ b/roles/docker/templates/compose-files/drone.yml.j2 @@ -2,7 +2,7 @@ version: "3.8" services: - drone: + app: image: drone/drone:{{ services.drone.version }} restart: unless-stopped networks: diff --git a/roles/docker/templates/compose-files/element.yml.j2 b/roles/docker/templates/compose-files/element.yml.j2 index 2a875cec..5f615eac 100644 --- a/roles/docker/templates/compose-files/element.yml.j2 +++ b/roles/docker/templates/compose-files/element.yml.j2 @@ -2,7 +2,7 @@ version: "3.8" services: - element: + app: image: avhost/docker-matrix-element:{{ services.element.version }} restart: unless-stopped networks: diff --git a/roles/docker/templates/compose-files/forgejo.yml.j2 b/roles/docker/templates/compose-files/forgejo.yml.j2 new file mode 100644 index 00000000..530b4632 --- /dev/null +++ b/roles/docker/templates/compose-files/forgejo.yml.j2 @@ -0,0 +1,37 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + app: + image: codeberg.org/forgejo/forgejo:{{ services.forgejo.version }} + restart: unless-stopped + networks: + - external_services + - postfix + volumes: + - ".:/data" + ports: + - "22:22" + environment: + VIRTUAL_HOST: "{{ services.forgejo.domain }}" + VIRTUAL_PORT: "3000" + LETSENCRYPT_HOST: "{{ services.forgejo.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + # Forgejo customization, see: https://docs.gitea.io/en-us/install-with-docker/#customization + # https://docs.gitea.io/en-us/config-cheat-sheet/#security-security + FORGEJO__mailer__ENABLED: true + FORGEJO__mailer__FROM: noreply@{{ services.forgejo.domain }} + FORGEJO__mailer__PROTOCOL: smtp + FORGEJO__mailer__SMTP_ADDR: "{{ smtp_host }}:{{ smtp_port }}" + FORGEJO__security__LOGIN_REMEMBER_DAYS: "60" + FORGEJO__security__PASSWORD_COMPLEXITY: off + FORGEJO__security__MIN_PASSWORD_LENGTH: "8" + FORGEJO__security__PASSWORD_CHECK_PWN: true + FORGEJO__service__ENABLE_NOTIFY_MAIL: true + FORGEJO__service__REGISTER_EMAIL_CONFIRM: true + +networks: + external_services: + external: true + postfix: + external: true diff --git a/roles/docker/templates/compose-files/nginx_proxy.yml.j2 b/roles/docker/templates/compose-files/nginx_proxy.yml.j2 new file mode 100644 index 00000000..ffee37a3 --- /dev/null +++ b/roles/docker/templates/compose-files/nginx_proxy.yml.j2 @@ -0,0 +1,38 @@ +version: "3.8" + +services: + proxy: + image: nginxproxy/nginx-proxy:{{ services.nginx_proxy.version }} + restart: always + networks: + - external_services + ports: + - "80:80" + - "443:443" + volumes: + - "./conf:/etc/nginx/conf.d" + - "./vhost:/etc/nginx/vhost.d" + - "./html:/usr/share/nginx/html" + - "./dhparam:/etc/nginx/dhparam" + - "./certs:/etc/nginx/certs:ro" + - "/var/run/docker.sock:/tmp/docker.sock:ro" + labels: + - com.github.nginx-proxy.nginx + +{% if letsencrypt_enabled %} + acme: + image: nginxproxy/acme-companion:{{ services.nginx_proxy.acme_companion_version }} + restart: always + volumes: + - "./vhost:/etc/nginx/vhost.d" + - "./html:/usr/share/nginx/html" + - "./dhparam:/etc/nginx/dhparam:ro" + - "./certs:/etc/nginx/certs" + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - proxy +{% endif %} + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/openldap.yml.j2 b/roles/docker/templates/compose-files/openldap.yml.j2 new file mode 100644 index 00000000..6d9532f9 --- /dev/null +++ b/roles/docker/templates/compose-files/openldap.yml.j2 @@ -0,0 +1,58 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + app: + image: osixia/openldap:{{ services.openldap.version }} + restart: unless-stopped + tty: true + stdin_open: true + volumes: + - "./var/lib/ldap:/var/lib/ldap" + - "./etc/slapd.d:/etc/ldap/slapd.d" + - "./certs:/container/service/slapd/assets/certs/" + ports: + - "389:389" + - "636:636" + hostname: "{{ services.openldap.domain }}" + domainname: "{{ services.openldap.domain }}" # important: same as hostname + environment: + LDAP_LOG_LEVEL: "256" + LDAP_ORGANISATION: "{{ base_domain }}" + LDAP_DOMAIN: "{{ base_domain }}" + LDAP_BASE_DN: "" + LDAP_ADMIN_PASSWORD: "{{ ldap_admin_password }}" + LDAP_CONFIG_PASSWORD: "{{ ldap_config_password }}" + LDAP_READONLY_USER: false + LDAP_RFC2307BIS_SCHEMA: false + LDAP_BACKEND: mdb + LDAP_TLS: true + LDAP_TLS_CRT_FILENAME: ldap.crt + LDAP_TLS_KEY_FILENAME: ldap.key + LDAP_TLS_CA_CRT_FILENAME: ca.crt + LDAP_TLS_ENFORCE: false + LDAP_TLS_CIPHER_SUITE: SECURE256:-VERS-SSL3.0 + LDAP_TLS_PROTOCOL_MIN: "3.1" + LDAP_TLS_VERIFY_CLIENT: demand + LDAP_REPLICATION: false + KEEP_EXISTING_CONFIG: false + LDAP_REMOVE_CONFIG_AFTER_SETUP: true + LDAP_SSL_HELPER_PREFIX: ldap + + admin: + image: osixia/phpldapadmin:{{ services.openldap.phpldapadmin_version }} + restart: unless-stopped + networks: + - default + - external_services + environment: + PHPLDAPADMIN_LDAP_HOSTS: app + PHPLDAPADMIN_HTTPS: false + PHPLDAPADMIN_TRUST_PROXY_SSL: true + VIRTUAL_HOST: "{{ services.openldap.domain }}" + LETSENCRYPT_HOST: "{{ services.openldap.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/portainer.yml.j2 b/roles/docker/templates/compose-files/portainer.yml.j2 new file mode 100644 index 00000000..5bbba8c5 --- /dev/null +++ b/roles/docker/templates/compose-files/portainer.yml.j2 @@ -0,0 +1,21 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + app: + image: portainer/portainer-ee:{{ services.portainer.version }} + restart: always + networks: + - external_services + volumes: + - ".:/data" + - "/var/run/docker.sock:/var/run/docker.sock:rw" + environment: + VIRTUAL_HOST: "{{ services.portainer.domain }}" + VIRTUAL_PORT: "9000" + LETSENCRYPT_HOST: "{{ services.portainer.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/postfix.yml.j2 b/roles/docker/templates/compose-files/postfix.yml.j2 new file mode 100644 index 00000000..89f25ba6 --- /dev/null +++ b/roles/docker/templates/compose-files/postfix.yml.j2 @@ -0,0 +1,20 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + app: + image: boky/postfix:{{ services.postfix.version }} + restart: always + networks: + - postfix + volumes: + - "./dkim:/etc/opendkim/keys" + environment: + # Get all services which have allowed_sender_domain defined + ALLOWED_SENDER_DOMAINS: "{{ services | dict2items | selectattr('value.allowed_sender_domain', 'true') | map(attribute='value.domain') | join(' ') }}" + HOSTNAME: "{{ services.postfix.domain }}" # the name the smtp server will identify itself as + DKIM_AUTOGENERATE: true + +networks: + postfix: + external: true diff --git a/roles/docker/templates/compose-files/privatebin.yml.j2 b/roles/docker/templates/compose-files/privatebin.yml.j2 new file mode 100644 index 00000000..717515d4 --- /dev/null +++ b/roles/docker/templates/compose-files/privatebin.yml.j2 @@ -0,0 +1,20 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + app: + image: jgeusebroek/privatebin:{{ services.privatebin.version }} + restart: unless-stopped + volumes: + - "./cfg:/privatebin/cfg" + - "./data:/privatebin/data" + networks: + - external_services + environment: + VIRTUAL_HOST: "{{ services.privatebin.domain }}" + LETSENCRYPT_HOST: "{{ services.privatebin.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/rallly.yml.j2 b/roles/docker/templates/compose-files/rallly.yml.j2 index f8cf987c..11a912d1 100644 --- a/roles/docker/templates/compose-files/rallly.yml.j2 +++ b/roles/docker/templates/compose-files/rallly.yml.j2 @@ -17,7 +17,7 @@ services: timeout: 5s retries: 5 - rallly: + app: image: lukevella/rallly:{{ services.rallly.version }} restart: always networks: diff --git a/roles/docker/templates/compose-files/watchtower.yml.j2 b/roles/docker/templates/compose-files/watchtower.yml.j2 new file mode 100644 index 00000000..642b6aa2 --- /dev/null +++ b/roles/docker/templates/compose-files/watchtower.yml.j2 @@ -0,0 +1,12 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + app: + image: containrrr/watchtower:{{ services.watchtower.version }} + restart: unless-stopped + environment: + WATCHTOWER_POLL_INTERVAL: "60" + volumes: + - "/root/.docker/config.json:/config.json:ro" + - "/var/run/docker.sock:/var/run/docker.sock" From 52b1d1ccd2f44d7b79c6649dadc47a7235c46d71 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Tue, 3 Oct 2023 21:19:51 +0200 Subject: [PATCH 05/23] Use a block to deploy all services + add pre_deploy and post_deploy --- roles/docker/defaults/main.yml | 49 +++++------- .../tasks/post_deploy/docker_registry.yml | 13 ++++ roles/docker/tasks/post_deploy/mastodon.yml | 19 +++++ .../tasks/pre_deploy/docker_registry.yml | 17 +++++ .../{services => pre_deploy}/element.yml | 17 +---- roles/docker/tasks/pre_deploy/hedgedoc.yml | 17 +++++ .../tasks/{services => pre_deploy}/mailu.yml | 19 +---- roles/docker/tasks/pre_deploy/mastodon.yml | 45 +++++++++++ .../tasks/{services => pre_deploy}/matrix.yml | 25 ++----- roles/docker/tasks/pre_deploy/nextcloud.yml | 17 +++++ roles/docker/tasks/pre_deploy/nginx_proxy.yml | 14 ++++ roles/docker/tasks/pre_deploy/openldap.yml | 12 +++ roles/docker/tasks/pre_deploy/postfix.yml | 13 ++++ roles/docker/tasks/pre_deploy/privatebin.yml | 16 ++++ roles/docker/tasks/pre_deploy/rallly.yml | 11 +++ .../tasks/{services => pre_deploy}/restic.yml | 17 +---- roles/docker/tasks/services.yml | 31 ++++++++ .../docker/tasks/services/docker_registry.yml | 40 ---------- roles/docker/tasks/services/drone.yml | 17 ----- roles/docker/tasks/services/forgejo.yml | 17 ----- roles/docker/tasks/services/hedgedoc.yml | 28 ------- roles/docker/tasks/services/keycloak.yml | 17 ----- roles/docker/tasks/services/mastodon.yml | 75 ------------------- roles/docker/tasks/services/membersystem.yml | 17 ----- roles/docker/tasks/services/netdata.yml | 17 ----- roles/docker/tasks/services/nextcloud.yml | 28 ------- roles/docker/tasks/services/nginx_proxy.yml | 25 ------- roles/docker/tasks/services/openldap.yml | 23 ------ roles/docker/tasks/services/passit.yml | 19 ----- roles/docker/tasks/services/portainer.yml | 17 ----- roles/docker/tasks/services/postfix.yml | 24 ------ roles/docker/tasks/services/privatebin.yml | 27 ------- roles/docker/tasks/services/rallly.yml | 22 ------ roles/docker/tasks/services/watchtower.yml | 17 ----- 34 files changed, 260 insertions(+), 522 deletions(-) create mode 100644 roles/docker/tasks/post_deploy/docker_registry.yml create mode 100644 roles/docker/tasks/post_deploy/mastodon.yml create mode 100644 roles/docker/tasks/pre_deploy/docker_registry.yml rename roles/docker/tasks/{services => pre_deploy}/element.yml (55%) create mode 100644 roles/docker/tasks/pre_deploy/hedgedoc.yml rename roles/docker/tasks/{services => pre_deploy}/mailu.yml (68%) create mode 100644 roles/docker/tasks/pre_deploy/mastodon.yml rename roles/docker/tasks/{services => pre_deploy}/matrix.yml (57%) create mode 100644 roles/docker/tasks/pre_deploy/nextcloud.yml create mode 100644 roles/docker/tasks/pre_deploy/nginx_proxy.yml create mode 100644 roles/docker/tasks/pre_deploy/openldap.yml create mode 100644 roles/docker/tasks/pre_deploy/postfix.yml create mode 100644 roles/docker/tasks/pre_deploy/privatebin.yml create mode 100644 roles/docker/tasks/pre_deploy/rallly.yml rename roles/docker/tasks/{services => pre_deploy}/restic.yml (74%) delete mode 100644 roles/docker/tasks/services/docker_registry.yml delete mode 100644 roles/docker/tasks/services/drone.yml delete mode 100644 roles/docker/tasks/services/forgejo.yml delete mode 100644 roles/docker/tasks/services/hedgedoc.yml delete mode 100644 roles/docker/tasks/services/keycloak.yml delete mode 100644 roles/docker/tasks/services/mastodon.yml delete mode 100644 roles/docker/tasks/services/membersystem.yml delete mode 100644 roles/docker/tasks/services/netdata.yml delete mode 100644 roles/docker/tasks/services/nextcloud.yml delete mode 100644 roles/docker/tasks/services/nginx_proxy.yml delete mode 100644 roles/docker/tasks/services/openldap.yml delete mode 100644 roles/docker/tasks/services/passit.yml delete mode 100644 roles/docker/tasks/services/portainer.yml delete mode 100644 roles/docker/tasks/services/postfix.yml delete mode 100644 roles/docker/tasks/services/privatebin.yml delete mode 100644 roles/docker/tasks/services/rallly.yml delete mode 100644 roles/docker/tasks/services/watchtower.yml diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index e0365ce6..68587796 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -6,38 +6,35 @@ services: ### Internal services ### postfix: - file: postfix.yml domain: "smtp.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/postfix" + pre_deploy_tasks: true version: "v3.6.1-alpine" nginx_proxy: - file: nginx_proxy.yml volume_folder: "{{ volume_root_folder }}/nginx" + pre_deploy_tasks: true version: "1.3-alpine" acme_companion_version: "2.2" openldap: - file: openldap.yml domain: "ldap.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/openldap" + pre_deploy_tasks: true version: "1.5.0" phpldapadmin_version: "0.9.0" netdata: - file: netdata.yml domain: "netdata.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/netdata" version: "v1" portainer: - file: portainer.yml domain: "portainer.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/portainer" version: "2.19.0" keycloak: - file: keycloak.yml domain: sso.{{ base_domain }} volume_folder: "{{ volume_root_folder }}/keycloak" version: "22.0" @@ -45,19 +42,20 @@ services: allowed_sender_domain: true restic: - file: restic.yml + volume_folder: "{{ volume_root_folder }}/restic" + pre_deploy_tasks: true user: dc-user domain: rynkeby.skovgaard.tel host_key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBLGol2G+a87ssy0nu/STKBZSiGyhZhZKx/ujfe9IeFo - volume_folder: "{{ volume_root_folder }}/restic" repository: restic version: "1.7.0" disabled_in_vagrant: true docker_registry: - file: docker_registry.yml domain: "docker.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/docker-registry" + pre_deploy_tasks: true + post_deploy_tasks: true username: "docker" password: "{{ docker_password }}" version: "2" @@ -65,23 +63,21 @@ services: ### External services ### nextcloud: - file: nextcloud.yml domain: "cloud.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/nextcloud" + pre_deploy_tasks: true version: 27-apache postgres_version: "10" redis_version: 7-alpine allowed_sender_domain: true forgejo: - file: forgejo.yml domain: "git.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/forgejo" version: "1.20" allowed_sender_domain: true passit: - file: passit.yml domain: "passit.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/passit" version: stable @@ -89,34 +85,33 @@ services: allowed_sender_domain: true matrix: - file: matrix.yml domain: "matrix.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/matrix" + pre_deploy_tasks: true version: v1.90.0 postgres_version: 15-alpine allowed_sender_domain: true element: - file: element.yml domain: "element.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/element" + pre_deploy_tasks: true version: v1.11.43 privatebin: - file: privatebin.yml domain: "paste.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/privatebin" + pre_deploy_tasks: true version: "20221009" hedgedoc: - file: hedgedoc.yml domain: "pad.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/hedgedoc" + pre_deploy_tasks: true version: 1.9.9-alpine postgres_version: 10-alpine data_coop_website: - file: websites/data.coop.yml domain: "{{ base_domain }}" www_domain: "www.{{ base_domain }}" version: stable @@ -124,28 +119,23 @@ services: staging_version: staging slides_2022_website: - file: websites/2022.slides.data.coop.yml domain: "2022.slides.{{ base_domain }}" version: latest fedi_dk_website: - file: websites/fedi.dk.yaml domain: fedi.dk version: latest vhs_website: - file: websites/vhs.data.coop.yaml domain: vhs.data.coop version: latest cryptohagen_website: - file: websites/cryptohagen.dk.yml domains: - "cryptohagen.dk" - "www.cryptohagen.dk" ulovliglogning_website: - file: websites/ulovliglogning.dk.yml domains: - "ulovliglogning.dk" - "www.ulovliglogning.dk" @@ -153,44 +143,42 @@ services: - "www.ulovlig-logning.dk" cryptoaarhus_website: - file: websites/cryptoaarhus.dk.yml domains: - "cryptoaarhus.dk" - "www.cryptoaarhus.dk" drone: - file: drone.yml domain: "drone.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/drone" version: "1" mailu: - file: mailu.yml - version: "1.9" domain: "mail.{{ base_domain }}" + volume_folder: "{{ volume_root_folder }}/mailu" + pre_deploy_tasks: true dns: 192.168.203.254 subnet: 192.168.203.0/24 - volume_folder: "{{ volume_root_folder }}/mailu" + version: "1.9" mastodon: - file: mastodon.yml domain: "social.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/mastodon" + pre_deploy_tasks: true + post_deploy_tasks: true version: v4.2.0 postgres_version: 14-alpine redis_version: 6-alpine allowed_sender_domain: true rallly: - file: rallly.yml domain: "when.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/rallly" + pre_deploy_tasks: true version: "2" postgres_version: 14-alpine allowed_sender_domain: true membersystem: - file: membersystem.yml domain: "member.{{ base_domain }}" django_admins: "Vidir:valberg@orn.li" volume_folder: "{{ volume_root_folder }}/membersystem" @@ -199,6 +187,5 @@ services: allowed_sender_domain: true watchtower: - file: watchtower.yml volume_folder: "{{ volume_root_folder }}/watchtower" version: "1.5.3" diff --git a/roles/docker/tasks/post_deploy/docker_registry.yml b/roles/docker/tasks/post_deploy/docker_registry.yml new file mode 100644 index 00000000..10bc5612 --- /dev/null +++ b/roles/docker/tasks/post_deploy/docker_registry.yml @@ -0,0 +1,13 @@ +# vim: ft=yaml.ansible +--- +- name: Generate htpasswd file + shell: docker compose exec registry htpasswd -Bbn docker {{ docker_password }} > auth/htpasswd + args: + chdir: "{{ services.docker_registry.volume_folder }}" + creates: "{{ services.docker_registry.volume_folder }}/auth/htpasswd" + +- name: log in to registry + docker_login: + registry: "{{ 'docker.data.coop' if vagrant else services.docker_registry.domain }}" + username: docker + password: "{{ docker_password }}" diff --git a/roles/docker/tasks/post_deploy/mastodon.yml b/roles/docker/tasks/post_deploy/mastodon.yml new file mode 100644 index 00000000..790e2d84 --- /dev/null +++ b/roles/docker/tasks/post_deploy/mastodon.yml @@ -0,0 +1,19 @@ +# vim: ft=yaml.ansible +--- +- name: Configure cron job to remove old Mastodon media daily + cron: + name: Clean Mastodon media data older than a week + cron_file: ansible_mastodon_clean_media + job: docker exec mastodon_web_1 tootctl media remove --days 7 + special_time: daily + user: root + state: present + +- name: Configure cron job to remove old Mastodon preview cards daily + cron: + name: Clean Mastodon preview card data older than two weeks + cron_file: ansible_mastodon_clean_preview_cards + job: docker exec mastodon_web_1 tootctl preview_cards remove --days 14 + special_time: daily + user: root + state: present diff --git a/roles/docker/tasks/pre_deploy/docker_registry.yml b/roles/docker/tasks/pre_deploy/docker_registry.yml new file mode 100644 index 00000000..33fd2ff5 --- /dev/null +++ b/roles/docker/tasks/pre_deploy/docker_registry.yml @@ -0,0 +1,17 @@ +# vim: ft=yaml.ansible +--- +- name: Create subfolders + file: + path: "{{ services.docker_registry.volume_folder }}/{{ volume }}" + state: directory + loop: + - auth + - registry + loop_control: + loop_var: volume + +- name: Copy docker registry vhost configuration + copy: + src: vhost/docker_registry + dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.docker_registry.domain }}" + mode: "0644" diff --git a/roles/docker/tasks/services/element.yml b/roles/docker/tasks/pre_deploy/element.yml similarity index 55% rename from roles/docker/tasks/services/element.yml rename to roles/docker/tasks/pre_deploy/element.yml index b325bdf2..26e3b918 100644 --- a/roles/docker/tasks/services/element.yml +++ b/roles/docker/tasks/pre_deploy/element.yml @@ -1,16 +1,16 @@ # vim: ft=yaml.ansible --- -- name: Create Element volume folder +- name: Create subfolder file: name: "{{ services.element.volume_folder }}/data" state: directory -- name: Upload Element config.json +- name: Upload config.json template: src: element/config.json.j2 dest: "{{ services.element.volume_folder }}/data/config.json" -- name: Upload Element riot.im.conf +- name: Upload riot.im.conf copy: src: element/riot.im.conf dest: "{{ services.element.volume_folder }}/data/riot.im.conf" @@ -19,14 +19,3 @@ copy: src: vhost/element dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.element.domain }}" - -- name: Upload Compose file for Element - template: - src: compose-files/element.yml.j2 - dest: "{{ services.element.volume_folder }}/docker-compose.yml" - -- name: Deploy Element - docker_compose: - project_src: "{{ services.element.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/pre_deploy/hedgedoc.yml b/roles/docker/tasks/pre_deploy/hedgedoc.yml new file mode 100644 index 00000000..d849ed3c --- /dev/null +++ b/roles/docker/tasks/pre_deploy/hedgedoc.yml @@ -0,0 +1,17 @@ +# vim: ft=yaml.ansible +--- +- name: Create subfolders + file: + name: "{{ services.hedgedoc.volume_folder }}/{{ volume }}" + state: directory + loop: + - db + - hedgedoc/uploads + loop_control: + loop_var: volume + +- name: Copy SSO certificate + copy: + src: sso/sso.data.coop.pem + dest: "{{ services.hedgedoc.volume_folder }}/sso.data.coop.pem" + mode: "0644" diff --git a/roles/docker/tasks/services/mailu.yml b/roles/docker/tasks/pre_deploy/mailu.yml similarity index 68% rename from roles/docker/tasks/services/mailu.yml rename to roles/docker/tasks/pre_deploy/mailu.yml index de4916d9..4dc1d5a3 100644 --- a/roles/docker/tasks/services/mailu.yml +++ b/roles/docker/tasks/pre_deploy/mailu.yml @@ -1,6 +1,6 @@ # vim: ft=yaml.ansible --- -- name: create mailu volume folders +- name: Create subfolders file: name: "{{ services.mailu.volume_folder }}/{{ volume }}" state: directory @@ -23,12 +23,12 @@ loop_control: loop_var: volume -- name: upload mailu.env file +- name: Upload mailu.env file template: src: mailu/env.j2 dest: "{{ services.mailu.volume_folder }}/mailu.env" -- name: hard link to Let's Encrypt TLS certificate +- name: Hard link to Let's Encrypt TLS certificate file: src: "{{ services.nginx_proxy.volume_folder }}/certs/{{ services.mailu.domain }}/fullchain.pem" dest: "{{ services.mailu.volume_folder }}/certs/cert.pem" @@ -36,21 +36,10 @@ force: true when: letsencrypt_enabled -- name: hard link to Let's Encrypt TLS key +- name: Hard link to Let's Encrypt TLS key file: src: "{{ services.nginx_proxy.volume_folder }}/certs/{{ services.mailu.domain }}/key.pem" dest: "{{ services.mailu.volume_folder }}/certs/key.pem" state: hard force: true when: letsencrypt_enabled - -- name: Upload Compose file for for Mailu - template: - src: compose-files/mailu.yml.j2 - dest: "{{ services.mailu.volume_folder }}/docker-compose.yml" - -- name: Deploy Mailu - docker_compose: - project_src: "{{ services.mailu.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/pre_deploy/mastodon.yml b/roles/docker/tasks/pre_deploy/mastodon.yml new file mode 100644 index 00000000..c32d9e17 --- /dev/null +++ b/roles/docker/tasks/pre_deploy/mastodon.yml @@ -0,0 +1,45 @@ +# vim: ft=yaml.ansible +--- +- name: Create subfolder for Mastodon data + file: + name: "{{ services.mastodon.volume_folder }}/mastodon_data" + state: directory + owner: "991" + mode: u=rwx,g=rx,o=rx + +- name: Create subfolder for PostgreSQL data + file: + name: "{{ services.mastodon.volume_folder }}/postgres_data" + state: directory + owner: "70" + mode: u=rwx,go= + +- name: Create subfolder for PostgreSQL config + file: + name: "{{ services.mastodon.volume_folder }}/postgres_config" + state: directory + owner: root + mode: u=rwx,g=rx,o=rx + +- name: Create subfolder for Redis data + file: + name: "{{ services.mastodon.volume_folder }}/redis_data" + state: directory + owner: "999" + group: "1000" + mode: u=rwx,g=rx,o=rx + +- name: Upload mastodon.env file + template: + src: mastodon/env.j2 + dest: "{{ services.mastodon.volume_folder }}/mastodon.env" + +- name: Upload vhost config for Mastodon domain + copy: + src: vhost/mastodon + dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.mastodon.domain }}" + +- name: Upload PostgreSQL config + copy: + src: mastodon/postgresql.conf + dest: "{{ services.mastodon.volume_folder }}/postgres_config/postgresql.conf" diff --git a/roles/docker/tasks/services/matrix.yml b/roles/docker/tasks/pre_deploy/matrix.yml similarity index 57% rename from roles/docker/tasks/services/matrix.yml rename to roles/docker/tasks/pre_deploy/matrix.yml index 2a4a6f8d..32ce95ed 100644 --- a/roles/docker/tasks/services/matrix.yml +++ b/roles/docker/tasks/pre_deploy/matrix.yml @@ -1,24 +1,24 @@ # vim: ft=yaml.ansible --- -- name: Create Matrix volume folders +- name: Create subfolders file: name: "{{ services.matrix.volume_folder }}/{{ volume }}" state: directory owner: "991" group: "991" loop: - - "data" - - "data/uploads" - - "data/media" + - data + - data/uploads + - data/media loop_control: loop_var: volume -- name: Create Matrix DB folder +- name: Create Matrix DB subfolder file: name: "{{ services.matrix.volume_folder }}/db" state: directory -- name: upload vhost config for matrix domain +- name: Upload vhost config for Matrix domain copy: src: vhost/matrix dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.matrix.domain }}" @@ -28,18 +28,7 @@ src: matrix/homeserver.yaml.j2 dest: "{{ services.matrix.volume_folder }}/data/homeserver.yaml" -- name: upload matrix logging config +- name: Upload Matrix logging config copy: src: matrix/log.config dest: "{{ services.matrix.volume_folder }}/data/matrix.data.coop.log.config" - -- name: Upload Compose file for Matrix - template: - src: compose-files/matrix.yml.j2 - dest: "{{ services.matrix.volume_folder }}/docker-compose.yml" - -- name: Deploy Matrix - docker_compose: - project_src: "{{ services.matrix.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/pre_deploy/nextcloud.yml b/roles/docker/tasks/pre_deploy/nextcloud.yml new file mode 100644 index 00000000..5a8e90e7 --- /dev/null +++ b/roles/docker/tasks/pre_deploy/nextcloud.yml @@ -0,0 +1,17 @@ +# vim: ft=yaml.ansible +--- +- name: Create subfolders + file: + path: "{{ services.nextcloud.volume_folder }}/{{ volume }}" + state: directory + loop: + - app + - postgres + loop_control: + loop_var: volume + +- name: Upload vhost config for Nextcloud domain + copy: + src: vhost/nextcloud + dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.nextcloud.domain }}" + notify: "restart nginx" diff --git a/roles/docker/tasks/pre_deploy/nginx_proxy.yml b/roles/docker/tasks/pre_deploy/nginx_proxy.yml new file mode 100644 index 00000000..be9e9d28 --- /dev/null +++ b/roles/docker/tasks/pre_deploy/nginx_proxy.yml @@ -0,0 +1,14 @@ +# vim: ft=yaml.ansible +--- +- name: Create subfolders + file: + name: "{{ services.nginx_proxy.volume_folder }}/{{ volume }}" + state: directory + loop: + - conf + - vhost + - html + - dhparam + - certs + loop_control: + loop_var: volume diff --git a/roles/docker/tasks/pre_deploy/openldap.yml b/roles/docker/tasks/pre_deploy/openldap.yml new file mode 100644 index 00000000..188c0628 --- /dev/null +++ b/roles/docker/tasks/pre_deploy/openldap.yml @@ -0,0 +1,12 @@ +# vim: ft=yaml.ansible +--- +- name: Create subfolders + file: + name: "{{ services.openldap.volume_folder }}/{{ volume }}" + state: directory + loop: + - var/lib/ldap + - etc/slapd + - certs + loop_control: + loop_var: volume diff --git a/roles/docker/tasks/pre_deploy/postfix.yml b/roles/docker/tasks/pre_deploy/postfix.yml new file mode 100644 index 00000000..e8b41fd3 --- /dev/null +++ b/roles/docker/tasks/pre_deploy/postfix.yml @@ -0,0 +1,13 @@ +# vim: ft=yaml.ansible +--- +- name: Set up network for Postfix + docker_network: + name: postfix + ipam_config: + - subnet: '172.16.0.0/16' + gateway: 172.16.0.1 + +- name: Create subfolder + file: + name: "{{ services.postfix.volume_folder }}/dkim" + state: directory diff --git a/roles/docker/tasks/pre_deploy/privatebin.yml b/roles/docker/tasks/pre_deploy/privatebin.yml new file mode 100644 index 00000000..012bd0b0 --- /dev/null +++ b/roles/docker/tasks/pre_deploy/privatebin.yml @@ -0,0 +1,16 @@ +# vim: ft=yaml.ansible +--- +- name: Create subfolders + file: + name: "{{ services.privatebin.volume_folder }}/{{ volume }}" + state: directory + loop: + - cfg + - data + loop_control: + loop_var: volume + +- name: Upload PrivateBin config + copy: + src: privatebin/conf.php + dest: "{{ services.privatebin.volume_folder }}/cfg/conf.php" diff --git a/roles/docker/tasks/pre_deploy/rallly.yml b/roles/docker/tasks/pre_deploy/rallly.yml new file mode 100644 index 00000000..3e91f9d5 --- /dev/null +++ b/roles/docker/tasks/pre_deploy/rallly.yml @@ -0,0 +1,11 @@ +# vim: ft=yaml.ansible +--- +- name: Create subfolder + file: + name: "{{ services.rallly.volume_folder }}/postgres" + state: directory + +- name: Copy rallly.env file + template: + src: rallly/env.j2 + dest: "{{ services.rallly.volume_folder }}/rallly.env" diff --git a/roles/docker/tasks/services/restic.yml b/roles/docker/tasks/pre_deploy/restic.yml similarity index 74% rename from roles/docker/tasks/services/restic.yml rename to roles/docker/tasks/pre_deploy/restic.yml index c838e265..8a147d77 100644 --- a/roles/docker/tasks/services/restic.yml +++ b/roles/docker/tasks/pre_deploy/restic.yml @@ -8,7 +8,7 @@ mode: '0755' state: directory -- name: Copy private SSH key +- name: Upload private SSH key copy: dest: "{{ services.restic.volume_folder }}/ssh/id_ed25519" owner: root @@ -31,7 +31,7 @@ mode: '0644' state: touch -- name: Create SSH config +- name: Upload SSH config template: src: restic/ssh.config.j2 dest: "{{ services.restic.volume_folder }}/ssh/config" @@ -39,21 +39,10 @@ group: root mode: '0600' -- name: Create SSH known_hosts file +- name: Upload SSH known_hosts file template: src: restic/ssh.known_hosts.j2 dest: "{{ services.restic.volume_folder }}/ssh/known_hosts" owner: root group: root mode: '0600' - -- name: Upload Compose file for Restic - template: - src: compose-files/restic.yml.j2 - dest: "{{ services.restic.volume_folder }}/docker-compose.yml" - -- name: Deploy Restic - docker_compose: - project_src: "{{ services.restic.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services.yml b/roles/docker/tasks/services.yml index 3b441e98..833554a0 100644 --- a/roles/docker/tasks/services.yml +++ b/roles/docker/tasks/services.yml @@ -4,6 +4,37 @@ docker_network: name: external_services +- name: Service block + loop: "{{ services | dict2items(key_name='name', value_name='vars') }}" + loop_control: + loop_var: service + when: single_service is not defined and + (service.vars.disabled_in_vagrant is not defined or + not (service.vars.disabled_in_vagrant and vagrant)) + block: + - name: Create volume folder + file: + name: "{{ service.vars.volume_folder }}" + state: directory + + - name: Upload Compose file + template: + src: compose-files/{{ service.name }}.yml.j2 + dest: "{{ service.vars.volume_folder }}/docker-compose.yml" + + - name: Run pre-deployment tasks + include_tasks: pre_deploy/{{ service.name }}.yml + when: service.vars.pre_deploy_tasks is defined and service.pre_deploy_tasks + + - name: Deploy Compose stack + command: docker compose up -d --remove-orphans --pull always + args: + chdir: "{{ service.vars.volume_folder }}" + + - name: Run post-deployment tasks + include_tasks: post_deploy/{{ service.name }}.yml + when: service.vars.post_deploy_tasks is defined and service.post_deploy_tasks + - name: setup services include_tasks: "services/{{ item.service.file }}" loop: "{{ services | dict2items(value_name='service') }}" diff --git a/roles/docker/tasks/services/docker_registry.yml b/roles/docker/tasks/services/docker_registry.yml deleted file mode 100644 index 3ef95429..00000000 --- a/roles/docker/tasks/services/docker_registry.yml +++ /dev/null @@ -1,40 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create Docker registry volume folders - file: - path: "{{ services.docker_registry.volume_folder }}/{{ volume }}" - state: directory - loop: - - auth - - registry - loop_control: - loop_var: volume - -- name: Copy docker registry vhost configuration - copy: - src: vhost/docker_registry - dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.docker_registry.domain }}" - mode: "0644" - -- name: Upload Compose file for Docker registry - template: - src: compose-files/docker_registry.yml.j2 - dest: "{{ services.docker_registry.volume_folder }}/docker-compose.yml" - -- name: Deploy Docker registry - docker_compose: - project_src: "{{ services.docker_registry.volume_folder }}" - pull: true - state: present - -- name: Generate htpasswd file - shell: "docker compose exec registry htpasswd -Bbn docker {{ docker_password }} > auth/htpasswd" - args: - chdir: "{{ services.docker_registry.volume_folder }}" - creates: "{{ services.docker_registry.volume_folder }}/auth/htpasswd" - -- name: log in to registry - docker_login: - registry: "{{ 'docker.data.coop' if vagrant else services.docker_registry.domain }}" - username: "docker" - password: "{{ docker_password }}" diff --git a/roles/docker/tasks/services/drone.yml b/roles/docker/tasks/services/drone.yml deleted file mode 100644 index 8e4fa257..00000000 --- a/roles/docker/tasks/services/drone.yml +++ /dev/null @@ -1,17 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create Drone volume folder - file: - path: "{{ services.drone.volume_folder }}" - state: directory - -- name: Upload Compose file for Drone - template: - src: compose-files/drone.yml.j2 - dest: "{{ services.drone.volume_folder }}/docker-compose.yml" - -- name: Deploy Drone - docker_compose: - project_src: "{{ services.drone.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/forgejo.yml b/roles/docker/tasks/services/forgejo.yml deleted file mode 100644 index 826a190c..00000000 --- a/roles/docker/tasks/services/forgejo.yml +++ /dev/null @@ -1,17 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create Forgejo volume folder - file: - name: "{{ services.portainer.volume_folder }}" - state: directory - -- name: Upload Compose file for Forgejo - template: - src: compose-files/forgejo.yml.j2 - dest: "{{ services.forgejo.volume_folder }}/docker-compose.yml" - -- name: Deploy Forgejo - docker_compose: - project_src: "{{ services.forgejo.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/hedgedoc.yml b/roles/docker/tasks/services/hedgedoc.yml deleted file mode 100644 index 6e5c874a..00000000 --- a/roles/docker/tasks/services/hedgedoc.yml +++ /dev/null @@ -1,28 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: create hedgedoc volume folders - file: - name: "{{ services.hedgedoc.volume_folder }}/{{ volume }}" - state: directory - loop: - - "db" - - "hedgedoc/uploads" - loop_control: - loop_var: volume - -- name: copy sso public certificate - copy: - src: sso/sso.data.coop.pem - dest: "{{ services.hedgedoc.volume_folder }}/sso.data.coop.pem" - mode: "0644" - -- name: Upload Compose file for for HedgeDoc - template: - src: compose-files/hedgedoc.yml.j2 - dest: "{{ services.hedgedoc.volume_folder }}/docker-compose.yml" - -- name: setup hedgedoc - docker_compose: - project_src: "{{ services.hedgedoc.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/keycloak.yml b/roles/docker/tasks/services/keycloak.yml deleted file mode 100644 index ff341b9a..00000000 --- a/roles/docker/tasks/services/keycloak.yml +++ /dev/null @@ -1,17 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create Keycloak volume folder - file: - path: "{{ services.keycloak.volume_folder }}/data" - state: directory - -- name: Upload Compose file for for Keycloak - template: - src: compose-files/keycloak.yml.j2 - dest: "{{ services.keycloak.volume_folder }}/docker-compose.yml" - -- name: Deploy Keycloak - docker_compose: - project_src: "{{ services.keycloak.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/mastodon.yml b/roles/docker/tasks/services/mastodon.yml deleted file mode 100644 index 95a14c08..00000000 --- a/roles/docker/tasks/services/mastodon.yml +++ /dev/null @@ -1,75 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create volume folder for Mastodon data - file: - name: "{{ services.mastodon.volume_folder }}/mastodon_data" - state: directory - owner: "991" - mode: u=rwx,g=rx,o=rx - -- name: Create volume folder for PostgreSQL data - file: - name: "{{ services.mastodon.volume_folder }}/postgres_data" - state: directory - owner: "70" - mode: u=rwx,go= - -- name: Create volume folder for PostgreSQL config - file: - name: "{{ services.mastodon.volume_folder }}/postgres_config" - state: directory - owner: root - mode: u=rwx,g=rx,o=rx - -- name: Create volume folder for Redis data - file: - name: "{{ services.mastodon.volume_folder }}/redis_data" - state: directory - owner: "999" - group: "1000" - mode: u=rwx,g=rx,o=rx - -- name: Copy mastodon environment file - template: - src: mastodon/env.j2 - dest: "{{ services.mastodon.volume_folder }}/mastodon.env" - -- name: Upload vhost config for Mastodon domain - copy: - src: vhost/mastodon - dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.mastodon.domain }}" - -- name: Copy PostgreSQL config - copy: - src: mastodon/postgresql.conf - dest: "{{ services.mastodon.volume_folder }}/postgres_config/postgresql.conf" - -- name: Upload Compose file for Mastodon - template: - src: compose-files/mastodon.yml.j2 - dest: "{{ services.mastodon.volume_folder }}/docker-compose.yml" - -- name: Deploy Mastodon - docker_compose: - project_src: "{{ services.mastodon.volume_folder }}" - pull: true - restarted: true - state: present - -- name: Configure cron job to remove old Mastodon media daily - cron: - name: Clean Mastodon media data older than a week - cron_file: ansible_mastodon_clean_media - job: docker exec mastodon_web_1 tootctl media remove --days 7 - special_time: daily - user: root - state: present - -- name: Configure cron job to remove old Mastodon preview cards daily - cron: - name: Clean Mastodon preview card data older than two weeks - cron_file: ansible_mastodon_clean_preview_cards - job: docker exec mastodon_web_1 tootctl preview_cards remove --days 14 - special_time: daily - user: root - state: present diff --git a/roles/docker/tasks/services/membersystem.yml b/roles/docker/tasks/services/membersystem.yml deleted file mode 100644 index 357c1695..00000000 --- a/roles/docker/tasks/services/membersystem.yml +++ /dev/null @@ -1,17 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create Membersystem volume folder - file: - name: "{{ services.membersystem.volume_folder }}" - state: directory - -- name: Upload Compose file for Membersystem - template: - src: compose-files/membersystem.yml.j2 - dest: "{{ services.membersystem.volume_folder }}/docker-compose.yml" - -- name: Deploy Membersystem - docker_compose: - project_src: "{{ services.membersystem.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/netdata.yml b/roles/docker/tasks/services/netdata.yml deleted file mode 100644 index e5234b62..00000000 --- a/roles/docker/tasks/services/netdata.yml +++ /dev/null @@ -1,17 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create Netdata volume folder - file: - path: "{{ services.netdata.volume_folder }}" - state: directory - -- name: Upload Compose file for Netdata - template: - src: compose-files/netdata.yml.j2 - dest: "{{ services.netdata.volume_folder }}/docker-compose.yml" - -- name: Deploy Netdata - docker_compose: - project_src: "{{ services.netdata.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/nextcloud.yml b/roles/docker/tasks/services/nextcloud.yml deleted file mode 100644 index 7273bcf0..00000000 --- a/roles/docker/tasks/services/nextcloud.yml +++ /dev/null @@ -1,28 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create Nextcloud volume folders - file: - path: "{{ services.nextcloud.volume_folder }}/{{ volume }}" - state: directory - loop: - - app - - postgres - loop_control: - loop_var: volume - -- name: upload vhost config for cloud.data.coop - copy: - src: vhost/nextcloud - dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.nextcloud.domain }}" - notify: "restart nginx" - -- name: Upload Compose file for Nextcloud - template: - src: compose-files/nextcloud.yml.j2 - dest: "{{ services.nextcloud.volume_folder }}/docker-compose.yml" - -- name: Deploy Nextcloud - docker_compose: - project_src: "{{ services.nextcloud.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/nginx_proxy.yml b/roles/docker/tasks/services/nginx_proxy.yml deleted file mode 100644 index 68659529..00000000 --- a/roles/docker/tasks/services/nginx_proxy.yml +++ /dev/null @@ -1,25 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: create nginx-proxy volume folders - file: - name: "{{ services.nginx_proxy.volume_folder }}/{{ volume }}" - state: directory - loop: - - conf - - vhost - - html - - dhparam - - certs - loop_control: - loop_var: volume - -- name: Upload Compose file for nginx-proxy - template: - src: compose-files/nginx_proxy.yml.j2 - dest: "{{ services.nginx_proxy.volume_folder }}/docker-compose.yml" - -- name: Deploy nginx-proxy - docker_compose: - project_src: "{{ services.nginx_proxy.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/openldap.yml b/roles/docker/tasks/services/openldap.yml deleted file mode 100644 index b477a5bf..00000000 --- a/roles/docker/tasks/services/openldap.yml +++ /dev/null @@ -1,23 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create OpenLDAP volume folders - file: - name: "{{ services.openldap.volume_folder }}/{{ volume }}" - state: directory - loop: - - var/lib/ldap - - etc/slapd - - certs - loop_control: - loop_var: volume - -- name: Upload Compose file for OpenLDAP - template: - src: compose-files/openldap.yml.j2 - dest: "{{ services.openldap.volume_folder }}/docker-compose.yml" - -- name: Deploy OpenLDAP - docker_compose: - project_src: "{{ services.openldap.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/passit.yml b/roles/docker/tasks/services/passit.yml deleted file mode 100644 index eaf5baad..00000000 --- a/roles/docker/tasks/services/passit.yml +++ /dev/null @@ -1,19 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create directory for Passit data - file: - name: "{{ services.passit.volume_folder }}/data" - owner: '70' - group: root - state: directory - -- name: Upload Compose file for Passit - template: - src: compose-files/passit.yml.j2 - dest: "{{ services.passit.volume_folder }}/docker-compose.yml" - -- name: Deploy Passit - docker_compose: - project_src: "{{ services.passit.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/portainer.yml b/roles/docker/tasks/services/portainer.yml deleted file mode 100644 index 5f158c91..00000000 --- a/roles/docker/tasks/services/portainer.yml +++ /dev/null @@ -1,17 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: create portainer volume folder - file: - name: "{{ services.portainer.volume_folder }}" - state: directory - -- name: Upload Compose file for Portainer - template: - src: compose-files/portainer.yml.j2 - dest: "{{ services.portainer.volume_folder }}/docker-compose.yml" - -- name: Deploy Portainer - docker_compose: - project_src: "{{ services.portainer.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/postfix.yml b/roles/docker/tasks/services/postfix.yml deleted file mode 100644 index 5298b15d..00000000 --- a/roles/docker/tasks/services/postfix.yml +++ /dev/null @@ -1,24 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Set up network for Postfix - docker_network: - name: postfix - ipam_config: - - subnet: '172.16.0.0/16' - gateway: 172.16.0.1 - -- name: Create volume folders for Postfix - file: - name: "{{ services.postfix.volume_folder }}/dkim" - state: directory - -- name: Upload Compose file for Postfix - template: - src: compose-files/postfix.yml.j2 - dest: "{{ services.forgejo.volume_folder }}/docker-compose.yml" - -- name: Deploy Postfix - docker_compose: - project_src: "{{ services.postfix.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/privatebin.yml b/roles/docker/tasks/services/privatebin.yml deleted file mode 100644 index 09d648ca..00000000 --- a/roles/docker/tasks/services/privatebin.yml +++ /dev/null @@ -1,27 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: create privatebin volume folders - file: - name: "{{ services.privatebin.volume_folder }}/{{ volume }}" - state: directory - loop: - - cfg - - data - loop_control: - loop_var: volume - -- name: upload privatebin config - copy: - src: privatebin/conf.php - dest: "{{ services.privatebin.volume_folder }}/cfg/conf.php" - -- name: Upload Compose file for PrivateBin - template: - src: compose-files/privatebin.yml.j2 - dest: "{{ services.privatebin.volume_folder }}/docker-compose.yml" - -- name: Deploy PrivateBin - docker_compose: - project_src: "{{ services.private.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/rallly.yml b/roles/docker/tasks/services/rallly.yml deleted file mode 100644 index e5f2b27b..00000000 --- a/roles/docker/tasks/services/rallly.yml +++ /dev/null @@ -1,22 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create Rallly volume folders - file: - name: "{{ services.rallly.volume_folder }}/postgres" - state: directory - -- name: Copy Rallly environment file - template: - src: rallly/env.j2 - dest: "{{ services.rallly.volume_folder }}/rallly.env" - -- name: Upload Compose file for Rallly - template: - src: compose-files/rallly.yml.j2 - dest: "{{ services.rallly.volume_folder }}/docker-compose.yml" - -- name: Deploy Rallly - docker_compose: - project_src: "{{ services.rallly.volume_folder }}" - pull: true - state: present diff --git a/roles/docker/tasks/services/watchtower.yml b/roles/docker/tasks/services/watchtower.yml deleted file mode 100644 index e528024d..00000000 --- a/roles/docker/tasks/services/watchtower.yml +++ /dev/null @@ -1,17 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Create Watchtower volume folder - file: - name: "{{ services.watchtower.volume_folder }}" - state: directory - -- name: Upload Compose file for Watchtower - template: - src: compose-files/watchtower.yml.j2 - dest: "{{ services.watchtower.volume_folder }}/docker-compose.yml" - -- name: Deploy Watchtower - docker_compose: - project_src: "{{ services.watchtower.volume_folder }}" - pull: true - state: present From f067a1b6c2294a8d72d7fa93a376af55c661eae9 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Tue, 3 Oct 2023 21:45:21 +0200 Subject: [PATCH 06/23] Convert websites to Compose stacks --- roles/docker/defaults/main.yml | 1 + .../tasks/pre_deploy/data_coop_website.yml | 11 +++++ .../websites/2022.slides.data.coop.yml | 19 -------- .../services/websites/cryptoaarhus.dk.yml | 13 ----- .../services/websites/cryptohagen.dk.yml | 13 ----- .../tasks/services/websites/data.coop.yml | 47 ------------------- .../services/websites/ulovliglogning.dk.yml | 13 ----- .../services/websites/vhs.data.coop.yaml | 19 -------- .../compose-files/cryptoaarhus_website.yml.j2 | 17 +++++++ .../compose-files/cryptohagen_website.yml.j2 | 17 +++++++ .../compose-files/data_coop_website.yml.j2 | 27 +++++++++++ .../compose-files/fedi_dk_website.yml.j2} | 25 +++++----- .../compose-files/slides_2022_website.yml.j2 | 22 +++++++++ .../ulovliglogning_website.yml.j2 | 17 +++++++ .../compose-files/vhs_website.yml.j2 | 22 +++++++++ 15 files changed, 148 insertions(+), 135 deletions(-) create mode 100644 roles/docker/tasks/pre_deploy/data_coop_website.yml delete mode 100644 roles/docker/tasks/services/websites/2022.slides.data.coop.yml delete mode 100644 roles/docker/tasks/services/websites/cryptoaarhus.dk.yml delete mode 100644 roles/docker/tasks/services/websites/cryptohagen.dk.yml delete mode 100644 roles/docker/tasks/services/websites/data.coop.yml delete mode 100644 roles/docker/tasks/services/websites/ulovliglogning.dk.yml delete mode 100644 roles/docker/tasks/services/websites/vhs.data.coop.yaml create mode 100644 roles/docker/templates/compose-files/cryptoaarhus_website.yml.j2 create mode 100644 roles/docker/templates/compose-files/cryptohagen_website.yml.j2 create mode 100644 roles/docker/templates/compose-files/data_coop_website.yml.j2 rename roles/docker/{tasks/services/websites/fedi.dk.yaml => templates/compose-files/fedi_dk_website.yml.j2} (51%) create mode 100644 roles/docker/templates/compose-files/slides_2022_website.yml.j2 create mode 100644 roles/docker/templates/compose-files/ulovliglogning_website.yml.j2 create mode 100644 roles/docker/templates/compose-files/vhs_website.yml.j2 diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 68587796..118d8dcc 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -114,6 +114,7 @@ services: data_coop_website: domain: "{{ base_domain }}" www_domain: "www.{{ base_domain }}" + pre_deploy_tasks: true version: stable staging_domain: "staging.{{ base_domain }}" staging_version: staging diff --git a/roles/docker/tasks/pre_deploy/data_coop_website.yml b/roles/docker/tasks/pre_deploy/data_coop_website.yml new file mode 100644 index 00000000..6ae6cbf4 --- /dev/null +++ b/roles/docker/tasks/pre_deploy/data_coop_website.yml @@ -0,0 +1,11 @@ +# vim: ft=yaml.ansible +--- +- name: Upload vhost config for root domain + copy: + src: vhost/base_domain + dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.data_coop_website.domain }}" + +- name: Upload vhost config for WWW domain + copy: + src: vhost/www.base_domain + dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.data_coop_website.www_domain }}" diff --git a/roles/docker/tasks/services/websites/2022.slides.data.coop.yml b/roles/docker/tasks/services/websites/2022.slides.data.coop.yml deleted file mode 100644 index 352b14a5..00000000 --- a/roles/docker/tasks/services/websites/2022.slides.data.coop.yml +++ /dev/null @@ -1,19 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: setup 2022.slides.data.coop website using unipi - docker_container: - name: 2022.slides.data.coop_website - image: docker.data.coop/unipi:{{ services.slides_2022_website.version }} - restart_policy: unless-stopped - purge_networks: yes - networks: - - name: external_services - env: - VIRTUAL_HOST: "{{ services.slides_2022_website.domain }}" - LETSENCRYPT_HOST: "{{ services.slides_2022_website.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - command: "--remote=https://git.data.coop/data.coop/slides.git#slides2022" - capabilities: - - NET_ADMIN - devices: - - "/dev/net/tun" diff --git a/roles/docker/tasks/services/websites/cryptoaarhus.dk.yml b/roles/docker/tasks/services/websites/cryptoaarhus.dk.yml deleted file mode 100644 index 648e8824..00000000 --- a/roles/docker/tasks/services/websites/cryptoaarhus.dk.yml +++ /dev/null @@ -1,13 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: setup cryptoaarhus.dk website docker container - docker_container: - name: cryptoaarhus_website - restart_policy: unless-stopped - image: docker.data.coop/cryptoaarhus-website - networks: - - name: external_services - env: - VIRTUAL_HOST : "{{ services.cryptoaarhus_website.domains|join(',') }}" - LETSENCRYPT_HOST: "{{ services.cryptoaarhus_website.domains|join(',') }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" diff --git a/roles/docker/tasks/services/websites/cryptohagen.dk.yml b/roles/docker/tasks/services/websites/cryptohagen.dk.yml deleted file mode 100644 index 655a06e0..00000000 --- a/roles/docker/tasks/services/websites/cryptohagen.dk.yml +++ /dev/null @@ -1,13 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: setup cryptohagen.dk website docker container - docker_container: - name: cryptohagen_website - restart_policy: unless-stopped - image: docker.data.coop/cryptohagen-website - networks: - - name: external_services - env: - VIRTUAL_HOST : "{{ services.cryptohagen_website.domains|join(',') }}" - LETSENCRYPT_HOST: "{{ services.cryptohagen_website.domains|join(',') }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" diff --git a/roles/docker/tasks/services/websites/data.coop.yml b/roles/docker/tasks/services/websites/data.coop.yml deleted file mode 100644 index 89eda566..00000000 --- a/roles/docker/tasks/services/websites/data.coop.yml +++ /dev/null @@ -1,47 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: Upload vhost config for root domain - copy: -<<<<<<< HEAD - src: vhost/base_domain - dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ base_domain }}" - -- name: Upload vhost config for WWW domain - copy: - src: vhost/www.base_domain - dest: "{{ services.nginx_proxy.volume_folder }}/vhost/www.{{ base_domain }}" -======= - src: files/configs/matrix/vhost-root - dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.data_coop_website.domain }}" - -- name: Upload vhost config for WWW domain - copy: - src: files/configs/vhost-www - dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.data_coop_website.www_domain }}" ->>>>>>> main - -- name: setup data.coop website docker container - docker_container: - name: "{{ services.data_coop_website.domain }}_website" - image: docker.data.coop/data-coop-website:{{ services.data_coop_website.version }} - pull: true - restart_policy: unless-stopped - networks: - - name: external_services - env: - VIRTUAL_HOST: "{{ services.data_coop_website.domain }},{{ services.data_coop_website.www_domain }}" - LETSENCRYPT_HOST: "{{ services.data_coop_website.domain }},{{ services.data_coop_website.www_domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - -- name: setup staging data.coop website using hugo - docker_container: - name: "{{ services.data_coop_website.staging_domain }}_website" - image: docker.data.coop/data-coop-website:{{ services.data_coop_website.staging_version }} - pull: true - restart_policy: unless-stopped - networks: - - name: external_services - env: - VIRTUAL_HOST: "{{ services.data_coop_website.staging_domain }}" - LETSENCRYPT_HOST: "{{ services.data_coop_website.staging_domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" diff --git a/roles/docker/tasks/services/websites/ulovliglogning.dk.yml b/roles/docker/tasks/services/websites/ulovliglogning.dk.yml deleted file mode 100644 index cc417899..00000000 --- a/roles/docker/tasks/services/websites/ulovliglogning.dk.yml +++ /dev/null @@ -1,13 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: setup ulovliglogning.dk website docker container - docker_container: - name: ulovliglogning_website - restart_policy: unless-stopped - image: ulovliglogning/ulovliglogning.dk:latest - networks: - - name: external_services - env: - VIRTUAL_HOST: "{{ services.ulovliglogning_website.domains|join(',') }}" - LETSENCRYPT_HOST: "{{ services.ulovliglogning_website.domains|join(',') }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" diff --git a/roles/docker/tasks/services/websites/vhs.data.coop.yaml b/roles/docker/tasks/services/websites/vhs.data.coop.yaml deleted file mode 100644 index f3b951a9..00000000 --- a/roles/docker/tasks/services/websites/vhs.data.coop.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# vim: ft=yaml.ansible ---- -- name: setup vhs.data.coop website with unipi - docker_container: - name: vhs.data.coop_website - image: docker.data.coop/unipi:{{ services.vhs_website.version }} - restart_policy: unless-stopped - purge_networks: yes - networks: - - name: external_services - env: - VIRTUAL_HOST: "{{ services.vhs_website.domain }}" - LETSENCRYPT_HOST: "{{ services.vhs_website.domain }}" - LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - command: "--remote=https://git.data.coop/vhs.data.coop/website.git#main" - capabilities: - - NET_ADMIN - devices: - - "/dev/net/tun" diff --git a/roles/docker/templates/compose-files/cryptoaarhus_website.yml.j2 b/roles/docker/templates/compose-files/cryptoaarhus_website.yml.j2 new file mode 100644 index 00000000..2a7d40a4 --- /dev/null +++ b/roles/docker/templates/compose-files/cryptoaarhus_website.yml.j2 @@ -0,0 +1,17 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + web: + image: docker.data.coop/cryptoaarhus-website + restart: unless-stopped + networks: + - external_services + environment: + VIRTUAL_HOST : "{{ services.cryptoaarhus_website.domains | join(',') }}" + LETSENCRYPT_HOST: "{{ services.cryptoaarhus_website.domains | join(',') }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/cryptohagen_website.yml.j2 b/roles/docker/templates/compose-files/cryptohagen_website.yml.j2 new file mode 100644 index 00000000..f8520f8c --- /dev/null +++ b/roles/docker/templates/compose-files/cryptohagen_website.yml.j2 @@ -0,0 +1,17 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + web: + image: docker.data.coop/cryptohagen-website + restart: unless-stopped + networks: + - external_services + environment: + VIRTUAL_HOST : "{{ services.cryptohagen_website.domains | join(',') }}" + LETSENCRYPT_HOST: "{{ services.cryptohagen_website.domains | join(',') }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/data_coop_website.yml.j2 b/roles/docker/templates/compose-files/data_coop_website.yml.j2 new file mode 100644 index 00000000..6910ca3b --- /dev/null +++ b/roles/docker/templates/compose-files/data_coop_website.yml.j2 @@ -0,0 +1,27 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + prod: + image: docker.data.coop/data-coop-website:{{ services.data_coop_website.version }} + restart: unless-stopped + networks: + - external_services + environment: + VIRTUAL_HOST: "{{ services.data_coop_website.domain }},{{ services.data_coop_website.www_domain }}" + LETSENCRYPT_HOST: "{{ services.data_coop_website.domain }},{{ services.data_coop_website.www_domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + + staging: + image: docker.data.coop/data-coop-website:{{ services.data_coop_website.staging_version }} + restart: unless-stopped + networks: + - external_services + environment: + VIRTUAL_HOST: "{{ services.data_coop_website.staging_domain }}" + LETSENCRYPT_HOST: "{{ services.data_coop_website.staging_domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + external_services: + external: true diff --git a/roles/docker/tasks/services/websites/fedi.dk.yaml b/roles/docker/templates/compose-files/fedi_dk_website.yml.j2 similarity index 51% rename from roles/docker/tasks/services/websites/fedi.dk.yaml rename to roles/docker/templates/compose-files/fedi_dk_website.yml.j2 index f4b97b27..94391572 100644 --- a/roles/docker/tasks/services/websites/fedi.dk.yaml +++ b/roles/docker/templates/compose-files/fedi_dk_website.yml.j2 @@ -1,19 +1,22 @@ -# vim: ft=yaml.ansible ---- -- name: setup fedi.dk website with unipi - docker_container: - name: fedi.dk_website +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + web: image: docker.data.coop/unipi:{{ services.fedi_dk_website.version }} - restart_policy: unless-stopped - purge_networks: yes + restart: unless-stopped networks: - - name: external_services - env: + - external_services + environment: VIRTUAL_HOST: "{{ services.fedi_dk_website.domain }}" LETSENCRYPT_HOST: "{{ services.fedi_dk_website.domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - command: "--remote=https://git.data.coop/fedi.dk/website.git#main" - capabilities: + command: --remote=https://git.data.coop/fedi.dk/website.git#main + cap_add: - NET_ADMIN devices: - "/dev/net/tun" + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/slides_2022_website.yml.j2 b/roles/docker/templates/compose-files/slides_2022_website.yml.j2 new file mode 100644 index 00000000..6a78119a --- /dev/null +++ b/roles/docker/templates/compose-files/slides_2022_website.yml.j2 @@ -0,0 +1,22 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + web: + image: docker.data.coop/unipi:{{ services.slides_2022_website.version }} + restart: unless-stopped + networks: + - external_services + environment: + VIRTUAL_HOST: "{{ services.slides_2022_website.domain }}" + LETSENCRYPT_HOST: "{{ services.slides_2022_website.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + command: --remote=https://git.data.coop/data.coop/slides.git#slides2022 + cap_add: + - NET_ADMIN + devices: + - "/dev/net/tun" + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/ulovliglogning_website.yml.j2 b/roles/docker/templates/compose-files/ulovliglogning_website.yml.j2 new file mode 100644 index 00000000..8cea97cc --- /dev/null +++ b/roles/docker/templates/compose-files/ulovliglogning_website.yml.j2 @@ -0,0 +1,17 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + web: + image: ulovliglogning/ulovliglogning.dk:latest + restart: unless-stopped + networks: + - external_services + environment: + VIRTUAL_HOST: "{{ services.ulovliglogning_website.domains | join(',') }}" + LETSENCRYPT_HOST: "{{ services.ulovliglogning_website.domains | join(',') }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + +networks: + external_services: + external: true diff --git a/roles/docker/templates/compose-files/vhs_website.yml.j2 b/roles/docker/templates/compose-files/vhs_website.yml.j2 new file mode 100644 index 00000000..077dcfca --- /dev/null +++ b/roles/docker/templates/compose-files/vhs_website.yml.j2 @@ -0,0 +1,22 @@ +# vim: ft=yaml.docker-compose +version: "3.8" + +services: + web: + image: docker.data.coop/unipi:{{ services.vhs_website.version }} + restart: unless-stopped + networks: + - external_services + environment: + VIRTUAL_HOST: "{{ services.vhs_website.domain }}" + LETSENCRYPT_HOST: "{{ services.vhs_website.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + command: --remote=https://git.data.coop/vhs.data.coop/website.git#main + cap_add: + - NET_ADMIN + devices: + - "/dev/net/tun" + +networks: + external_services: + external: true From 62f548d05be9afecd666916e9ca4cf5ea9273a40 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Tue, 3 Oct 2023 22:00:51 +0200 Subject: [PATCH 07/23] Fix task for single service --- roles/docker/tasks/services.yml | 41 ++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/roles/docker/tasks/services.yml b/roles/docker/tasks/services.yml index 833554a0..f170f1a2 100644 --- a/roles/docker/tasks/services.yml +++ b/roles/docker/tasks/services.yml @@ -4,7 +4,7 @@ docker_network: name: external_services -- name: Service block +- name: Service block for all services loop: "{{ services | dict2items(key_name='name', value_name='vars') }}" loop_control: loop_var: service @@ -24,7 +24,7 @@ - name: Run pre-deployment tasks include_tasks: pre_deploy/{{ service.name }}.yml - when: service.vars.pre_deploy_tasks is defined and service.pre_deploy_tasks + when: service.vars.pre_deploy_tasks is defined and service.vars.pre_deploy_tasks - name: Deploy Compose stack command: docker compose up -d --remove-orphans --pull always @@ -33,20 +33,33 @@ - name: Run post-deployment tasks include_tasks: post_deploy/{{ service.name }}.yml - when: service.vars.post_deploy_tasks is defined and service.post_deploy_tasks + when: service.vars.post_deploy_tasks is defined and service.vars.post_deploy_tasks -- name: setup services - include_tasks: "services/{{ item.service.file }}" - loop: "{{ services | dict2items(value_name='service') }}" - when: single_service is not defined and - item.service.file is defined and - (item.service.disabled_in_vagrant is not defined or - not (item.service.disabled_in_vagrant and vagrant)) - -- name: setup single service - include_tasks: "services/{{ services[single_service].file }}" +- name: Service block for a single service when: single_service is defined and single_service in services and - services[single_service].file is defined and (services[single_service].disabled_in_vagrant is not defined or not (services[single_service].disabled_in_vagrant and vagrant)) + block: + - name: Create volume folder + file: + name: "{{ services[single_service].volume_folder }}" + state: directory + + - name: Upload Compose file + template: + src: compose-files/{{ single_service }}.yml.j2 + dest: "{{ services[single_service].volume_folder }}/docker-compose.yml" + + - name: Run pre-deployment tasks + include_tasks: pre_deploy/{{ single_service }}.yml + when: services[single_service].pre_deploy_tasks is defined and services[single_service].pre_deploy_tasks + + - name: Deploy Compose stack + command: docker compose up -d --remove-orphans --pull always + args: + chdir: "{{ services[single_service].volume_folder }}" + + - name: Run post-deployment tasks + include_tasks: post_deploy/{{ single_service }}.yml + when: services[single_service].post_deploy_tasks is defined and services[single_service].post_deploy_tasks From 6cb06d43f1c66a1bf8aab5a9c0ee5c8467577fe2 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Tue, 3 Oct 2023 22:13:30 +0200 Subject: [PATCH 08/23] Formatting --- roles/docker/tasks/services.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/docker/tasks/services.yml b/roles/docker/tasks/services.yml index f170f1a2..433a4f56 100644 --- a/roles/docker/tasks/services.yml +++ b/roles/docker/tasks/services.yml @@ -36,8 +36,7 @@ when: service.vars.post_deploy_tasks is defined and service.vars.post_deploy_tasks - name: Service block for a single service - when: single_service is defined and - single_service in services and + when: single_service is defined and single_service in services and (services[single_service].disabled_in_vagrant is not defined or not (services[single_service].disabled_in_vagrant and vagrant)) block: From d0b23d4ef5258525cc77431e18906df26006d33c Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 18:37:57 +0200 Subject: [PATCH 09/23] Specify cpus in Vagrantfile --- Vagrantfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Vagrantfile b/Vagrantfile index 589d56cc..eb6bf038 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,6 +13,7 @@ Vagrant.configure(2) do |config| config.vm.hostname = "datacoop" config.vm.provider :virtualbox do |v| + v.cpus = 4 v.memory = 8192 end From f8b4e49f7f574c3b87d901f0fce8feb69d970667 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 18:42:56 +0200 Subject: [PATCH 10/23] Don't base 'vagrant' on virtualization (prep for Proxmox) --- Vagrantfile | 7 ++++++- playbook.yml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index eb6bf038..6a5b7ea9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -27,7 +27,12 @@ Vagrant.configure(2) do |config| if provisioned? config.ssh.guest_port = PORT ansible.extra_vars = { - ansible_port: PORT + ansible_port: PORT, + from_vagrant: true + } + else + ansible.extra_vars = { + from_vagrant: true } end end diff --git a/playbook.yml b/playbook.yml index d2ce5af8..de51a464 100644 --- a/playbook.yml +++ b/playbook.yml @@ -6,7 +6,7 @@ vars: ldap_dn: "dc=data,dc=coop" - vagrant: "{{ ansible_virtualization_role == 'guest' }}" + vagrant: "{{ from_vagrant is defined and from_vagrant }}" letsencrypt_enabled: "{{ not vagrant }}" base_domain: "{{ 'datacoop.devel' if vagrant else 'data.coop' }}" From 301d1b77190f760ed0f61c0d9f76d914869a30a1 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 19:35:09 +0200 Subject: [PATCH 11/23] Add missing volume_folder vars --- roles/docker/defaults/main.yml | 8 ++++++++ .../templates/compose-files/data_coop_website.yml.j2 | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 118d8dcc..6e539e83 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -1,6 +1,7 @@ # vim: ft=yaml.ansible --- volume_root_folder: "/docker-volumes" +volume_website_folder: "{{ volume_root_folder }}/websites" services: @@ -114,6 +115,7 @@ services: data_coop_website: domain: "{{ base_domain }}" www_domain: "www.{{ base_domain }}" + volume_folder: "{{ volume_website_folder }}/datacoop" pre_deploy_tasks: true version: stable staging_domain: "staging.{{ base_domain }}" @@ -121,20 +123,24 @@ services: slides_2022_website: domain: "2022.slides.{{ base_domain }}" + volume_folder: "{{ volume_website_folder }}/slides-2022" version: latest fedi_dk_website: domain: fedi.dk + volume_folder: "{{ volume_website_folder }}/fedidk" version: latest vhs_website: domain: vhs.data.coop + volume_folder: "{{ volume_website_folder }}/vhs" version: latest cryptohagen_website: domains: - "cryptohagen.dk" - "www.cryptohagen.dk" + volume_folder: "{{ volume_website_folder }}/cryptohagen" ulovliglogning_website: domains: @@ -142,11 +148,13 @@ services: - "www.ulovliglogning.dk" - "ulovlig-logning.dk" - "www.ulovlig-logning.dk" + volume_folder: "{{ volume_website_folder }}/ulovliglogning" cryptoaarhus_website: domains: - "cryptoaarhus.dk" - "www.cryptoaarhus.dk" + volume_folder: "{{ volume_website_folder }}/cryptoaarhus" drone: domain: "drone.{{ base_domain }}" diff --git a/roles/docker/templates/compose-files/data_coop_website.yml.j2 b/roles/docker/templates/compose-files/data_coop_website.yml.j2 index 6910ca3b..909cbecc 100644 --- a/roles/docker/templates/compose-files/data_coop_website.yml.j2 +++ b/roles/docker/templates/compose-files/data_coop_website.yml.j2 @@ -2,7 +2,7 @@ version: "3.8" services: - prod: + prod-web: image: docker.data.coop/data-coop-website:{{ services.data_coop_website.version }} restart: unless-stopped networks: @@ -12,7 +12,7 @@ services: LETSENCRYPT_HOST: "{{ services.data_coop_website.domain }},{{ services.data_coop_website.www_domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - staging: + staging-web: image: docker.data.coop/data-coop-website:{{ services.data_coop_website.staging_version }} restart: unless-stopped networks: From 3001317e20079b54fab4e9d7b58745f5789cc04a Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 19:35:52 +0200 Subject: [PATCH 12/23] Ansible doesn't support looping over a block --- roles/docker/tasks/loop.yml | 24 ++++++++++++++++++++++++ roles/docker/tasks/services.yml | 33 ++++++--------------------------- 2 files changed, 30 insertions(+), 27 deletions(-) create mode 100644 roles/docker/tasks/loop.yml diff --git a/roles/docker/tasks/loop.yml b/roles/docker/tasks/loop.yml new file mode 100644 index 00000000..9ad47734 --- /dev/null +++ b/roles/docker/tasks/loop.yml @@ -0,0 +1,24 @@ +# vim: ft=yaml.ansible +--- +- name: Create volume folder + file: + name: "{{ service.vars.volume_folder }}" + state: directory + +- name: Upload Compose file + template: + src: compose-files/{{ service.name }}.yml.j2 + dest: "{{ service.vars.volume_folder }}/docker-compose.yml" + +- name: Run pre-deployment tasks + include_tasks: pre_deploy/{{ service.name }}.yml + when: service.vars.pre_deploy_tasks is defined and service.vars.pre_deploy_tasks + +- name: Deploy Compose stack + command: docker compose up -d --remove-orphans --pull always + args: + chdir: "{{ service.vars.volume_folder }}" + +- name: Run post-deployment tasks + include_tasks: post_deploy/{{ service.name }}.yml + when: service.vars.post_deploy_tasks is defined and service.vars.post_deploy_tasks diff --git a/roles/docker/tasks/services.yml b/roles/docker/tasks/services.yml index 433a4f56..6943c514 100644 --- a/roles/docker/tasks/services.yml +++ b/roles/docker/tasks/services.yml @@ -5,35 +5,14 @@ name: external_services - name: Service block for all services + include_tasks: + file: loop.yml + vars: + service: "{{ item }}" loop: "{{ services | dict2items(key_name='name', value_name='vars') }}" - loop_control: - loop_var: service when: single_service is not defined and - (service.vars.disabled_in_vagrant is not defined or - not (service.vars.disabled_in_vagrant and vagrant)) - block: - - name: Create volume folder - file: - name: "{{ service.vars.volume_folder }}" - state: directory - - - name: Upload Compose file - template: - src: compose-files/{{ service.name }}.yml.j2 - dest: "{{ service.vars.volume_folder }}/docker-compose.yml" - - - name: Run pre-deployment tasks - include_tasks: pre_deploy/{{ service.name }}.yml - when: service.vars.pre_deploy_tasks is defined and service.vars.pre_deploy_tasks - - - name: Deploy Compose stack - command: docker compose up -d --remove-orphans --pull always - args: - chdir: "{{ service.vars.volume_folder }}" - - - name: Run post-deployment tasks - include_tasks: post_deploy/{{ service.name }}.yml - when: service.vars.post_deploy_tasks is defined and service.vars.post_deploy_tasks + (item.vars.disabled_in_vagrant is not defined or + not (item.vars.disabled_in_vagrant and vagrant)) - name: Service block for a single service when: single_service is defined and single_service in services and From 3ac2d839716bf6c8c91c8cbbe36ff24c31976672 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 19:43:11 +0200 Subject: [PATCH 13/23] Magic --- roles/docker/tasks/{loop.yml => block.yml} | 0 roles/docker/tasks/services.yml | 35 +++++----------------- 2 files changed, 8 insertions(+), 27 deletions(-) rename roles/docker/tasks/{loop.yml => block.yml} (100%) diff --git a/roles/docker/tasks/loop.yml b/roles/docker/tasks/block.yml similarity index 100% rename from roles/docker/tasks/loop.yml rename to roles/docker/tasks/block.yml diff --git a/roles/docker/tasks/services.yml b/roles/docker/tasks/services.yml index 6943c514..488b1dee 100644 --- a/roles/docker/tasks/services.yml +++ b/roles/docker/tasks/services.yml @@ -1,12 +1,12 @@ # vim: ft=yaml.ansible --- -- name: setup external services network +- name: Set up external services network docker_network: name: external_services -- name: Service block for all services +- name: Deploy all services include_tasks: - file: loop.yml + file: block.yml vars: service: "{{ item }}" loop: "{{ services | dict2items(key_name='name', value_name='vars') }}" @@ -14,30 +14,11 @@ (item.vars.disabled_in_vagrant is not defined or not (item.vars.disabled_in_vagrant and vagrant)) -- name: Service block for a single service +- name: Deploy single service + include_tasks: + file: block.yml + vars: + service: "{{ {single_service: services[single_service]} | dict2items(key_name='name', value_name='vars') }}" when: single_service is defined and single_service in services and (services[single_service].disabled_in_vagrant is not defined or not (services[single_service].disabled_in_vagrant and vagrant)) - block: - - name: Create volume folder - file: - name: "{{ services[single_service].volume_folder }}" - state: directory - - - name: Upload Compose file - template: - src: compose-files/{{ single_service }}.yml.j2 - dest: "{{ services[single_service].volume_folder }}/docker-compose.yml" - - - name: Run pre-deployment tasks - include_tasks: pre_deploy/{{ single_service }}.yml - when: services[single_service].pre_deploy_tasks is defined and services[single_service].pre_deploy_tasks - - - name: Deploy Compose stack - command: docker compose up -d --remove-orphans --pull always - args: - chdir: "{{ services[single_service].volume_folder }}" - - - name: Run post-deployment tasks - include_tasks: post_deploy/{{ single_service }}.yml - when: services[single_service].post_deploy_tasks is defined and services[single_service].post_deploy_tasks From 98fcc2d634713310c05d1fc5cafecf7b2aa7fdb5 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 19:44:39 +0200 Subject: [PATCH 14/23] Include service name in task names in block.yml --- roles/docker/tasks/block.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/docker/tasks/block.yml b/roles/docker/tasks/block.yml index 9ad47734..82da08e1 100644 --- a/roles/docker/tasks/block.yml +++ b/roles/docker/tasks/block.yml @@ -1,24 +1,24 @@ # vim: ft=yaml.ansible --- -- name: Create volume folder +- name: Create volume folder for service {{ service.name }} file: name: "{{ service.vars.volume_folder }}" state: directory -- name: Upload Compose file +- name: Upload Compose file for service {{ service.name }} template: src: compose-files/{{ service.name }}.yml.j2 dest: "{{ service.vars.volume_folder }}/docker-compose.yml" -- name: Run pre-deployment tasks +- name: Run pre-deployment tasks for service {{ service.name }} include_tasks: pre_deploy/{{ service.name }}.yml when: service.vars.pre_deploy_tasks is defined and service.vars.pre_deploy_tasks -- name: Deploy Compose stack +- name: Deploy Compose stack for service {{ service.name }} command: docker compose up -d --remove-orphans --pull always args: chdir: "{{ service.vars.volume_folder }}" -- name: Run post-deployment tasks +- name: Run post-deployment tasks for service {{ service.name }} include_tasks: post_deploy/{{ service.name }}.yml when: service.vars.post_deploy_tasks is defined and service.vars.post_deploy_tasks From af6a130695b92590dd1f2146f3ad07c7f1dbd645 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 19:57:31 +0200 Subject: [PATCH 15/23] Fix handler and name --- roles/docker/handlers/main.yml | 9 ++++----- roles/docker/templates/compose-files/netdata.yml.j2 | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/roles/docker/handlers/main.yml b/roles/docker/handlers/main.yml index ad671dd8..d6c78752 100644 --- a/roles/docker/handlers/main.yml +++ b/roles/docker/handlers/main.yml @@ -1,7 +1,6 @@ # vim: ft=yaml.ansible --- -- name: "restart nginx" - community.docker.docker_container: - name: "nginx-proxy" - restart: "yes" - state: "started" +- name: restart nginx + command: docker compose restart proxy + args: + chdir: "{{ services.nginx_proxy.volume_folder }}" diff --git a/roles/docker/templates/compose-files/netdata.yml.j2 b/roles/docker/templates/compose-files/netdata.yml.j2 index dcf5eada..34976659 100644 --- a/roles/docker/templates/compose-files/netdata.yml.j2 +++ b/roles/docker/templates/compose-files/netdata.yml.j2 @@ -24,7 +24,7 @@ services: security_opt: - apparmor:unconfined - socket_proxy: + socket-proxy: image: tecnativa/docker-socket-proxy:latest volumes: - "/var/run/docker.sock:/var/run/docker.sock:ro" From 3dc4e14c156572d75cb52c82cb9a094beb6a1753 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 19:59:09 +0200 Subject: [PATCH 16/23] Bump Vagrant specs --- Vagrantfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 6a5b7ea9..9a43d034 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,8 +13,8 @@ Vagrant.configure(2) do |config| config.vm.hostname = "datacoop" config.vm.provider :virtualbox do |v| - v.cpus = 4 - v.memory = 8192 + v.cpus = 8 + v.memory = 16384 end config.vm.provision :ansible do |ansible| From 5ae78bcd17e18fdaf679066b0a3f7d453054250d Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 21:34:59 +0200 Subject: [PATCH 17/23] Fix magic --- roles/docker/tasks/services.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/docker/tasks/services.yml b/roles/docker/tasks/services.yml index 488b1dee..676335cb 100644 --- a/roles/docker/tasks/services.yml +++ b/roles/docker/tasks/services.yml @@ -18,7 +18,7 @@ include_tasks: file: block.yml vars: - service: "{{ {single_service: services[single_service]} | dict2items(key_name='name', value_name='vars') }}" + service: "{{ {single_service: services[single_service]} | dict2items(key_name='name', value_name='vars') | join }}" when: single_service is defined and single_service in services and (services[single_service].disabled_in_vagrant is not defined or not (services[single_service].disabled_in_vagrant and vagrant)) From 2966e6715b33c430aae6335734b19d9932989d49 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 21:44:37 +0200 Subject: [PATCH 18/23] Add shell to users --- roles/ubuntu_base/tasks/users.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/ubuntu_base/tasks/users.yml b/roles/ubuntu_base/tasks/users.yml index 8ef07b60..2e6d3e21 100644 --- a/roles/ubuntu_base/tasks/users.yml +++ b/roles/ubuntu_base/tasks/users.yml @@ -6,7 +6,8 @@ comment: "{{ item.comment }}" password: "{{ item.password }}" groups: "{{ item.groups }}" - update_password: "always" + update_password: always + shell: /bin/bash loop: "{{ users | default([]) }}" - name: "Add ssh authorized_keys" From 15fa5d6215bfcec3c1694c3678e3d7e7736153e8 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 22:02:11 +0200 Subject: [PATCH 19/23] No need for Python Docker bindings since we use Docker cmd --- roles/docker/tasks/main.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 81b56eed..e4d2b58f 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -17,16 +17,6 @@ name: docker-ce state: present -- name: install docker python bindings - pip: - executable: pip3 - name: "{{ packages }}" - state: present - vars: - packages: - - docker - - docker-compose - - name: Configure cron job to prune unused Docker data weekly cron: name: Prune unused Docker data From 85e1da3cbf733ff4239b422ca2de7e3928c269eb Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Wed, 4 Oct 2023 22:05:59 +0200 Subject: [PATCH 20/23] Last fixes + install Compose v2 plugin --- roles/docker/tasks/main.yml | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index e4d2b58f..0a42ea22 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -1,21 +1,31 @@ # vim: ft=yaml.ansible --- -- name: add docker gpg key +- name: Add Docker PGP key apt_key: keyserver: pgp.mit.edu id: 8D81803C0EBFCD88 state: present -- name: add docker apt repository +- name: Add Docker apt repository apt_repository: repo: deb https://download.docker.com/linux/ubuntu bionic stable state: present update_cache: yes -- name: install docker-ce +- name: Install Docker apt: - name: docker-ce + name: "{{ pkgs }}" state: present + vars: + pkgs: + - docker-ce + - docker-compose-plugin + +- name: Create docker-compose symlink + ansible.builtin.file: + name: /usr/local/bin/docker-compose + src: /usr/libexec/docker/cli-plugins/docker-compose + state: link - name: Configure cron job to prune unused Docker data weekly cron: @@ -26,12 +36,15 @@ user: root state: present -- name: create folder structure for bind mounts +- name: Create folder structure for bind mounts file: - name: "{{ volume_root_folder }}" + name: "{{ item }}" state: directory + loop: + - "{{ volume_root_folder }}" + - "{{ volume_website_folder }}" -- name: setup services +- name: Set up services import_tasks: services.yml tags: - setup_services From 4082c6fde3601c805ddd339a89d578ca3ec9e9b5 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 4 Nov 2023 01:20:53 +0100 Subject: [PATCH 21/23] Add from_vagrant to deploy.sh --- deploy.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy.sh b/deploy.sh index ee107346..429e338f 100755 --- a/deploy.sh +++ b/deploy.sh @@ -13,6 +13,7 @@ BASE_CMD="ansible-playbook playbook.yml --ask-vault-pass" if [ "$1" = "--vagrant" ]; then BASE_CMD="$BASE_CMD --verbose --inventory=vagrant_host" + VAGRANT_VAR="from_vagrant" shift fi @@ -29,17 +30,17 @@ else "services") if [ -z "$2" ]; then echo "Deploying all services!" - $BASE_CMD --tags setup_services + eval "$BASE_CMD --tags setup_services $(test -z "$VAGRANT_VAR" || printf '%s' "$VAGRANT_VAR=true")" else echo "Deploying service: $2" - $BASE_CMD --tags setup_services --extra-vars "single_service=$2" + $BASE_CMD --tags setup_services --extra-vars '{"single_service": "'"$2"'"'"$(test -z "$VAGRANT_VAR" || printf '%s' ', "'"$VAGRANT_VAR"'": true')"'}' fi ;; "base") - $BASE_CMD --tags base_only + eval "$BASE_CMD --tags base_only $(test -z "$VAGRANT_VAR" || printf '%s' "$VAGRANT_VAR=true")" ;; "users") - $BASE_CMD --tags setup-users + eval "$BASE_CMD --tags setup-users $(test -z "$VAGRANT_VAR" || printf '%s' "$VAGRANT_VAR=true")" ;; *) usage From e426c3d6c5452e114902b36613e8f21f9a416fa8 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Thu, 7 Dec 2023 20:47:11 +0100 Subject: [PATCH 22/23] Rename Write Freely compose file --- .../compose-files/{writefreely.yml => writefreely.yml.j2} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename roles/docker/templates/compose-files/{writefreely.yml => writefreely.yml.j2} (100%) diff --git a/roles/docker/templates/compose-files/writefreely.yml b/roles/docker/templates/compose-files/writefreely.yml.j2 similarity index 100% rename from roles/docker/templates/compose-files/writefreely.yml rename to roles/docker/templates/compose-files/writefreely.yml.j2 From bd074929ac88ce5b6a533056fe204b21340907fa Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 9 Dec 2023 19:37:46 +0100 Subject: [PATCH 23/23] Fix stuff --- roles/docker/tasks/main.yml | 6 ------ roles/docker/templates/compose-files/nginx_proxy.yml.j2 | 2 +- roles/docker/templates/mailu/env.j2 | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 0a42ea22..8d297ea5 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -21,12 +21,6 @@ - docker-ce - docker-compose-plugin -- name: Create docker-compose symlink - ansible.builtin.file: - name: /usr/local/bin/docker-compose - src: /usr/libexec/docker/cli-plugins/docker-compose - state: link - - name: Configure cron job to prune unused Docker data weekly cron: name: Prune unused Docker data diff --git a/roles/docker/templates/compose-files/nginx_proxy.yml.j2 b/roles/docker/templates/compose-files/nginx_proxy.yml.j2 index ffee37a3..e811955c 100644 --- a/roles/docker/templates/compose-files/nginx_proxy.yml.j2 +++ b/roles/docker/templates/compose-files/nginx_proxy.yml.j2 @@ -17,7 +17,7 @@ services: - "./certs:/etc/nginx/certs:ro" - "/var/run/docker.sock:/tmp/docker.sock:ro" labels: - - com.github.nginx-proxy.nginx + - com.github.jrcs.letsencrypt_nginx_proxy_companion.nginx_proxy {% if letsencrypt_enabled %} acme: diff --git a/roles/docker/templates/mailu/env.j2 b/roles/docker/templates/mailu/env.j2 index 9b7f7822..ad0ab16c 100644 --- a/roles/docker/templates/mailu/env.j2 +++ b/roles/docker/templates/mailu/env.j2 @@ -127,7 +127,7 @@ WEBSITE=https://{{ services.mailu.domain }} # LOG_DRIVER=json-file # Docker-compose project name, this will prepended to containers names. -COMPOSE_PROJECT_NAME=mail_server +COMPOSE_PROJECT_NAME=mailu # Number of rounds used by the password hashing scheme CREDENTIAL_ROUNDS=12