From 787f47d45ec7182a59f046a460323f03b1addc7e Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Sun, 3 Mar 2019 07:38:00 +0100 Subject: [PATCH 01/12] Set restart policy on containers that were missing it --- roles/docker/tasks/services/netdata.yml | 1 + roles/docker/tasks/services/postfix.yml | 1 + roles/docker/tasks/services/websites.yml | 6 ++++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/docker/tasks/services/netdata.yml b/roles/docker/tasks/services/netdata.yml index ba08afb..adfa1ab 100644 --- a/roles/docker/tasks/services/netdata.yml +++ b/roles/docker/tasks/services/netdata.yml @@ -4,6 +4,7 @@ docker_container: name: netdata image: netdata/netdata + restart_policy: unless-stopped hostname: "hevonen.servers.{{ base_domain }}" capabilities: - SYS_PTRACE diff --git a/roles/docker/tasks/services/postfix.yml b/roles/docker/tasks/services/postfix.yml index 5d5d6db..59d5472 100644 --- a/roles/docker/tasks/services/postfix.yml +++ b/roles/docker/tasks/services/postfix.yml @@ -4,6 +4,7 @@ docker_container: name: postfix image: boky/postfix + restart_policy: unless-stopped networks: - name: postfix env: diff --git a/roles/docker/tasks/services/websites.yml b/roles/docker/tasks/services/websites.yml index 2b95692..5833414 100644 --- a/roles/docker/tasks/services/websites.yml +++ b/roles/docker/tasks/services/websites.yml @@ -2,8 +2,9 @@ - name: setup data.coop website docker container docker_container: - name: website + name: data.coop_website image: docker.data.coop/data-coop-website + restart_policy: unless-stopped networks: - name: external_services env: @@ -13,7 +14,8 @@ - name: setup cryptohagen.dk website docker container docker_container: - name: website + name: cryptohagen_website + restart_policy: unless-stopped image: docker.data.coop/cryptohagen-website networks: - name: external_services From 0c1e94323cc5fee79aa1d1aff2cd4fe1b9d3e39a Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Sat, 2 Mar 2019 22:34:33 +0100 Subject: [PATCH 02/12] Add drone CI/CD pipeline --- playbook.yml | 1 + roles/docker/defaults/main.yml | 4 ++++ roles/docker/tasks/services/drone.yml | 21 +++++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 roles/docker/tasks/services/drone.yml diff --git a/playbook.yml b/playbook.yml index 7c82f5d..8b7f99a 100644 --- a/playbook.yml +++ b/playbook.yml @@ -21,6 +21,7 @@ - codimd - netdata - docker_registry + - drone - websites smtp_host: postfix diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 74d66fa..75e3683 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -48,3 +48,7 @@ data_coop_website: cryptohagen_website: domain: "cryptohagen.dk" + +drone: + domain: "drone.{{ base_domain }}" + volume_folder: "{{ volume_root_folder }}/drone" diff --git a/roles/docker/tasks/services/drone.yml b/roles/docker/tasks/services/drone.yml new file mode 100644 index 0000000..ad05c46 --- /dev/null +++ b/roles/docker/tasks/services/drone.yml @@ -0,0 +1,21 @@ +--- +- name: Drone container + docker_container: + name: drone + image: drone/drone:latest + restart_policy: unless-stopped + networks: + - name: external_services + volumes: + - "{{ drone.volume_folder }}:/data" + - "/var/run/docker.sock:/var/run/docker.sock" + env: + DRONE_GITEA_SERVER: "https://{{ gitea.domain }}" + DRONE_GITEA_ALWAYS_AUTH: "False" + DRONE_RUNNER_CAPACITY: "2" + DRONE_SERVER_HOST: "{{ drone.domain }}" + DRONE_SERVER_PROTO: "https" + PLUGIN_CUSTOM_DNS: "91.239.100.100" + VIRTUAL_HOST: "{{ drone.domain }}" + LETSENCRYPT_HOST: "{{ drone.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" From 9ff11808ce4bd55e0826a9a4a0beda413eed6ea9 Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Sun, 3 Mar 2019 15:26:39 +0100 Subject: [PATCH 03/12] Add watchtower to manage auto-update of containers --- playbook.yml | 1 + roles/docker/tasks/services/watchtower.yml | 12 ++++++++++++ roles/docker/tasks/services/websites.yml | 4 ++++ 3 files changed, 17 insertions(+) create mode 100644 roles/docker/tasks/services/watchtower.yml diff --git a/playbook.yml b/playbook.yml index 8b7f99a..db999e1 100644 --- a/playbook.yml +++ b/playbook.yml @@ -23,6 +23,7 @@ - docker_registry - drone - websites + - watchtower smtp_host: postfix smtp_port: 587 diff --git a/roles/docker/tasks/services/watchtower.yml b/roles/docker/tasks/services/watchtower.yml new file mode 100644 index 0000000..ffe05af --- /dev/null +++ b/roles/docker/tasks/services/watchtower.yml @@ -0,0 +1,12 @@ +--- +- name: watchtower container + docker_container: + name: watchtower + image: v2tec/watchtower + restart_policy: unless-stopped + networks: + - name: external_services + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /root/.docker/config.json:/config.json + command: ["--label-enable"] diff --git a/roles/docker/tasks/services/websites.yml b/roles/docker/tasks/services/websites.yml index 5833414..74649cd 100644 --- a/roles/docker/tasks/services/websites.yml +++ b/roles/docker/tasks/services/websites.yml @@ -11,6 +11,8 @@ VIRTUAL_HOST : "{{ data_coop_website.domain }}" LETSENCRYPT_HOST: "{{ data_coop_website.domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + labels: + com.centurylinklabs.watchtower.enable: "true" - name: setup cryptohagen.dk website docker container docker_container: @@ -23,3 +25,5 @@ VIRTUAL_HOST : "{{ cryptohagen_website.domain }}" LETSENCRYPT_HOST: "{{ cryptohagen_website.domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + labels: + com.centurylinklabs.watchtower.enable: "true" From 2f413b3e99cc0e1b0e1cffe2923f4981625413a1 Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Mon, 4 Mar 2019 16:28:51 +0100 Subject: [PATCH 04/12] Switch out watchtower with Ouroboros --- playbook.yml | 2 +- roles/docker/tasks/services/ouroboros.yml | 18 ++++++++++++++++++ roles/docker/tasks/services/watchtower.yml | 12 ------------ roles/docker/tasks/services/websites.yml | 4 ++-- 4 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 roles/docker/tasks/services/ouroboros.yml delete mode 100644 roles/docker/tasks/services/watchtower.yml diff --git a/playbook.yml b/playbook.yml index db999e1..c953aa3 100644 --- a/playbook.yml +++ b/playbook.yml @@ -23,7 +23,7 @@ - docker_registry - drone - websites - - watchtower + - ouroboros smtp_host: postfix smtp_port: 587 diff --git a/roles/docker/tasks/services/ouroboros.yml b/roles/docker/tasks/services/ouroboros.yml new file mode 100644 index 0000000..c5aae9f --- /dev/null +++ b/roles/docker/tasks/services/ouroboros.yml @@ -0,0 +1,18 @@ +--- +- name: ouroboros container + docker_container: + name: ouroboros + image: pyouroboros/ouroboros + restart_policy: unless-stopped + networks: + - name: external_services + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /root/.docker/config.json:/root/.docker/config.json + env: + LABEL_ENABLE: "true" + LABELS_ONLY: "true" + CLEANUP: "true" + LATEST: "true" + CRON: "*/1 * * * *" + \ No newline at end of file diff --git a/roles/docker/tasks/services/watchtower.yml b/roles/docker/tasks/services/watchtower.yml deleted file mode 100644 index ffe05af..0000000 --- a/roles/docker/tasks/services/watchtower.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: watchtower container - docker_container: - name: watchtower - image: v2tec/watchtower - restart_policy: unless-stopped - networks: - - name: external_services - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - /root/.docker/config.json:/config.json - command: ["--label-enable"] diff --git a/roles/docker/tasks/services/websites.yml b/roles/docker/tasks/services/websites.yml index 74649cd..2436da2 100644 --- a/roles/docker/tasks/services/websites.yml +++ b/roles/docker/tasks/services/websites.yml @@ -12,7 +12,7 @@ LETSENCRYPT_HOST: "{{ data_coop_website.domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" labels: - com.centurylinklabs.watchtower.enable: "true" + com.ouroboros.enable: "true" - name: setup cryptohagen.dk website docker container docker_container: @@ -26,4 +26,4 @@ LETSENCRYPT_HOST: "{{ cryptohagen_website.domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" labels: - com.centurylinklabs.watchtower.enable: "true" + com.ouroboros.enable: "true" From 55c8e77254d6e6364adcd03c6201e7968619d205 Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Mon, 4 Mar 2019 18:21:14 +0100 Subject: [PATCH 05/12] Move openldap to volume mounts --- roles/docker/defaults/main.yml | 1 + roles/docker/tasks/services/openldap.yml | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 75e3683..3d0368d 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -3,6 +3,7 @@ nginx: ldap: domain: "ldap.{{ base_domain }}" + volume_folder: "{{ volume_root_folder }}/openldap" thelounge: domain: "irc.{{ base_domain }}" diff --git a/roles/docker/tasks/services/openldap.yml b/roles/docker/tasks/services/openldap.yml index dcdfef5..463ac0b 100644 --- a/roles/docker/tasks/services/openldap.yml +++ b/roles/docker/tasks/services/openldap.yml @@ -1,4 +1,14 @@ --- +- name: create ldap volume folders + file: + name: "{{ ldap.volume_folder }}/{{ volume }}" + state: directory + loop: + - "var/lib/ldap" + - "etc/slapd" + - "certs" + loop_control: + loop_var: volume - name: Create a network for ldap docker_network: @@ -11,9 +21,9 @@ tty: true interactive: true volumes: - - /var/lib/ldap - - /etc/ldap/slapd.d - - /container/service/slapd/assets/certs/ + - "{{ ldap.volume_folder }}/var/lib/ldap:/var/lib/ldap" + - "{{ ldap.volume_folder }}/etc/slapd.d:/etc/ldap/slapd.d" + - "{{ ldap.volume_folder }}/certs:/container/service/slapd/assets/certs/" published_ports: - "389:389" - "636:636" From 1f8b1827ff71299240a0ca90cd45c31f58893f0e Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Mon, 4 Mar 2019 21:38:36 +0100 Subject: [PATCH 06/12] Rearrange matrix+riot ansible script, move volumes to host mounts. --- roles/docker/defaults/main.yml | 2 + .../{riot-config.json => riot/config.json} | 2 +- .../files/configs/{ => riot}/riot.im.conf | 0 roles/docker/tasks/services/matrix_riot.yml | 45 ++++++++++--------- 4 files changed, 28 insertions(+), 21 deletions(-) rename roles/docker/files/configs/{riot-config.json => riot/config.json} (96%) rename roles/docker/files/configs/{ => riot}/riot.im.conf (100%) diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 3d0368d..7c97ede 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -23,9 +23,11 @@ fider: matrix: domain: "matrix.{{ base_domain }}" + volume_folder: "{{ volume_root_folder }}/matrix" riot: domain: "riot.{{ base_domain }}" + volume_folder: "{{ volume_root_folder }}/riot" privatebin: domain: "paste.{{ base_domain }}" diff --git a/roles/docker/files/configs/riot-config.json b/roles/docker/files/configs/riot/config.json similarity index 96% rename from roles/docker/files/configs/riot-config.json rename to roles/docker/files/configs/riot/config.json index c3b5ca7..a7dbfc9 100644 --- a/roles/docker/files/configs/riot-config.json +++ b/roles/docker/files/configs/riot/config.json @@ -23,7 +23,7 @@ "feature_tabbed_settings": "enable", "feature_sas": "enable" }, - "welcomeUserId": "@riot-bot:matrix.org", + "welcomeUserId": "", "piwik": false, "roomDirectory": { "servers": [ diff --git a/roles/docker/files/configs/riot.im.conf b/roles/docker/files/configs/riot/riot.im.conf similarity index 100% rename from roles/docker/files/configs/riot.im.conf rename to roles/docker/files/configs/riot/riot.im.conf diff --git a/roles/docker/tasks/services/matrix_riot.yml b/roles/docker/tasks/services/matrix_riot.yml index 8579a58..93fbdeb 100644 --- a/roles/docker/tasks/services/matrix_riot.yml +++ b/roles/docker/tasks/services/matrix_riot.yml @@ -1,26 +1,35 @@ --- +- name: create matrix volume folders + file: + name: "{{ matrix.volume_folder }}/{{ volume }}" + state: directory + loop: + - "db" + loop_control: + loop_var: volume + +- name: create riot volume folders + file: + name: "{{ riot.volume_folder }}/{{ volume }}" + state: directory + loop: + - "data" + loop_control: + loop_var: volume - name: matrix network docker_network: name: matrix -- name: matrix database volume - docker_volume: - name: matrix_db - -- name: riot volume - docker_volume: - name: riot_app - - name: upload riot config.json template: - src: files/configs/riot-config.json - dest: /var/lib/docker/volumes/riot_app/_data/config.json + src: files/configs/riot/config.json + dest: "{{ riot.volume_folder }}/data/config.json" - name: upload riot.im.conf template: - src: files/configs/riot.im.conf - dest: /var/lib/docker/volumes/riot_app/_data/riot.im.conf + src: files/configs/riot/riot.im.conf + dest: "{{ riot.volume_folder }}/data/riot.im.conf" - name: matrix database container docker_container: @@ -31,7 +40,7 @@ networks: - name: matrix volumes: - - matrix_db:/var/lib/postgresql/data + - "{{ matrix.volume_folder }}/db:/var/lib/postgresql/data" env: POSTGRES_USER: "synapse" POSTGRES_PASSWORD: "{{ postgres_passwords.matrix }}" @@ -43,9 +52,6 @@ restart_policy: unless-stopped networks: - name: matrix - published_ports: - - 8008:8008 - - 8448:8448 env: SYNAPSE_SERVER_NAME: "{{ base_domain }}" SYNAPSE_REPORT_STATS: "False" @@ -69,13 +75,12 @@ networks: - name: matrix - name: external_services - volumes: - - riot_app:/data published_ports: - - 8080 + - "8080" + volumes: + - "{{ riot.volume_folder }}/data:/data" env: VIRTUAL_HOST: "{{ riot.domain }}" VIRTUAL_PORT: "8080" LETSENCRYPT_HOST: "{{ riot.domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" - From fef1951d57d155763da51b09900927fa81dba842 Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Tue, 5 Mar 2019 08:28:53 +0100 Subject: [PATCH 07/12] Add necessary nginx configs to get matrix federation to work --- roles/docker/files/configs/matrix/vhost-matrix | 1 + roles/docker/files/configs/matrix/vhost-root | 14 ++++++++++++++ roles/docker/tasks/services/matrix_riot.yml | 5 +++++ 3 files changed, 20 insertions(+) create mode 100644 roles/docker/files/configs/matrix/vhost-matrix create mode 100644 roles/docker/files/configs/matrix/vhost-root diff --git a/roles/docker/files/configs/matrix/vhost-matrix b/roles/docker/files/configs/matrix/vhost-matrix new file mode 100644 index 0000000..36b8434 --- /dev/null +++ b/roles/docker/files/configs/matrix/vhost-matrix @@ -0,0 +1 @@ +listen 8008; \ No newline at end of file diff --git a/roles/docker/files/configs/matrix/vhost-root b/roles/docker/files/configs/matrix/vhost-root new file mode 100644 index 0000000..5e814d2 --- /dev/null +++ b/roles/docker/files/configs/matrix/vhost-root @@ -0,0 +1,14 @@ +location /_matrix { + proxy_pass http://0.0.0.0:8008; + proxy_set_header X-Forwarded-For $remote_addr; +} + +location /.well-known/matrix/server { + default_type application/json; + return 200 '{"m.server": "matrix.data.coop:443"}'; +} + +location /.well-known/matrix/client { + default_type application/json; + return 200 '{"m.homeserver": {"base_url": "https://matrix.data.coop"}}'; +} diff --git a/roles/docker/tasks/services/matrix_riot.yml b/roles/docker/tasks/services/matrix_riot.yml index 93fbdeb..9a95722 100644 --- a/roles/docker/tasks/services/matrix_riot.yml +++ b/roles/docker/tasks/services/matrix_riot.yml @@ -31,6 +31,11 @@ src: files/configs/riot/riot.im.conf dest: "{{ riot.volume_folder }}/data/riot.im.conf" +- name: upload vhost config for root domain + template: + src: files/configs/matrix/vhost-root + dest: "{{ nginx.volume_folder }}/vhost/{{ base_domain }}" + - name: matrix database container docker_container: name: matrix_db From 4db622313dbb3f09d59dd28bbd9e2b952f077b1a Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Tue, 5 Mar 2019 09:04:47 +0100 Subject: [PATCH 08/12] Publish port for nginx to forward connections to --- roles/docker/tasks/services/matrix_riot.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/docker/tasks/services/matrix_riot.yml b/roles/docker/tasks/services/matrix_riot.yml index 9a95722..3339b07 100644 --- a/roles/docker/tasks/services/matrix_riot.yml +++ b/roles/docker/tasks/services/matrix_riot.yml @@ -57,6 +57,8 @@ restart_policy: unless-stopped networks: - name: matrix + published_ports: + - "8008" env: SYNAPSE_SERVER_NAME: "{{ base_domain }}" SYNAPSE_REPORT_STATS: "False" From ae2873e4d967d365485ad005798afebbfe389ac5 Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Tue, 5 Mar 2019 09:06:19 +0100 Subject: [PATCH 09/12] vhost config file for matrix domain --- roles/docker/tasks/services/matrix_riot.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/docker/tasks/services/matrix_riot.yml b/roles/docker/tasks/services/matrix_riot.yml index 3339b07..68c0320 100644 --- a/roles/docker/tasks/services/matrix_riot.yml +++ b/roles/docker/tasks/services/matrix_riot.yml @@ -36,6 +36,11 @@ src: files/configs/matrix/vhost-root dest: "{{ nginx.volume_folder }}/vhost/{{ base_domain }}" +- name: upload vhost config for matrix domain + template: + src: files/configs/matrix/vhost-matrix + dest: "{{ nginx.volume_folder }}/vhost/{{ matrix.domain }}" + - name: matrix database container docker_container: name: matrix_db From 0c5ed4860089260ac0c296d128f0717f673f0383 Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Tue, 5 Mar 2019 10:06:10 +0100 Subject: [PATCH 10/12] Upgrade CodiMD --- roles/docker/tasks/services/codimd.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/docker/tasks/services/codimd.yml b/roles/docker/tasks/services/codimd.yml index d58a0b9..1e0e950 100644 --- a/roles/docker/tasks/services/codimd.yml +++ b/roles/docker/tasks/services/codimd.yml @@ -32,7 +32,7 @@ - name: codimd app container docker_container: name: codimd_app - image: hackmdio/hackmd:1.2.1 + image: hackmdio/hackmd:1.3.0 restart_policy: unless-stopped networks: - name: codimd @@ -51,6 +51,7 @@ CMD_LDAP_BINDCREDENTIALS: "{{ ldap_admin_password }}" CMD_LDAP_SEARCHBASE: "dc=data,dc=coop" CMD_LDAP_SEARCHFILTER: "(&(uid={{ '{{username}}' }})(objectClass=inetOrgPerson))" + CMD_USECDN: "false" VIRTUAL_HOST: "{{ codimd.domain }}" LETSENCRYPT_HOST: "{{ codimd.domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" From d5602af9998b5a578b907e7f9850fead99f25e85 Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Tue, 5 Mar 2019 10:31:31 +0100 Subject: [PATCH 11/12] Add haveged to base system packages --- roles/ubuntu_base/tasks/base.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/ubuntu_base/tasks/base.yml b/roles/ubuntu_base/tasks/base.yml index 8851eb5..d6a59c8 100644 --- a/roles/ubuntu_base/tasks/base.yml +++ b/roles/ubuntu_base/tasks/base.yml @@ -7,4 +7,5 @@ - aptitude - python3-pip - apparmor + - haveged \ No newline at end of file From 0f398cef3f8e37d8868a9493b118bf95d73c08e2 Mon Sep 17 00:00:00 2001 From: Jesper Hess Nielsen Date: Tue, 5 Mar 2019 10:59:32 +0100 Subject: [PATCH 12/12] Upgrade riot to 1.0.1 --- roles/docker/tasks/services/matrix_riot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/docker/tasks/services/matrix_riot.yml b/roles/docker/tasks/services/matrix_riot.yml index 68c0320..d17768c 100644 --- a/roles/docker/tasks/services/matrix_riot.yml +++ b/roles/docker/tasks/services/matrix_riot.yml @@ -81,7 +81,7 @@ - name: riot container docker_container: name: riot_app - image: avhost/docker-matrix-riot:v1.0.0 + image: avhost/docker-matrix-riot:v1.0.1 state: started restart_policy: always networks: