From 62d5a3cccab5f16c486aa092d8a039e75c5ee609 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Fri, 25 Nov 2022 23:31:30 +0100 Subject: [PATCH 01/24] Add README.md --- README.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..bafd8c3a --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ +# data.coop infrastructure + +This repository contains the code used to deploy data.coop's services +and websites. We use Ansible to encode our infrastructure setup. Only +the association's administrators have access to deploy the services. + +## Deploying + +To deploy the services, the included `deploy.sh` script can be used. The +Ansible playbook uses two custom-made roles (in the `roles/` directory). +The script has options to deploy only one of the roles, which are namely +`ubuntu_base` and `docker`, the latter of which is for deploying the +services themselves using Docker containers. Select services only can +also be specified. By default, the script deploys everything. + +Here is a summary of the options that can be used: + +```console +$ ./deploy.sh # deploy everything +$ ./deploy.sh base # deploy the ubuntu_base role only +$ ./deploy.sh services # deploy the docker role only +$ ./deploy.sh services SERVICE # deploy SERVICE only +``` + +`SERVICE` should match one of the filenames in +`roles/docker/tasks/services/`, but without the `.yml` extension + +## Testing + +In order for us to be able to test our setup locally, we use Vagrant to +deploy the services in a virtual machine. To do this, Vagrant and +VirtualBox must both be installed on the development machine. Then, the +services can be deployed locally by using the `vagrant` command-line +tool. The working directory needs to be the root of the repository for +this to work properly. + +> Note: As our secrets are contained in an Ansible Vault file, only the +> administrators have the ability to run the deployment in Vagrant. +> However, one could replace the vault file for testing purposes. + +Here is a summary of the commands that are available with the `vagrant` +command-line tool: + +```console +$ vagrant up # Create and provision the VM +$ vagrant provision # Re-provision the VM +$ vagrant ssh # SSH into the VM +$ vagrant halt # Power down the VM +$ vagrant destroy # Power down and delete the VM +``` + From 4611d890f7ef3db0ba946c89fa72b22fa74eee0a Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 26 Nov 2022 16:32:06 +0100 Subject: [PATCH 02/24] Update README.md --- README.md | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bafd8c3a..114d1a1d 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,17 @@ The script has options to deploy only one of the roles, which are namely services themselves using Docker containers. Select services only can also be specified. By default, the script deploys everything. -Here is a summary of the options that can be used: +Here is a summary of the options that can be used with the script: ```console -$ ./deploy.sh # deploy everything -$ ./deploy.sh base # deploy the ubuntu_base role only -$ ./deploy.sh services # deploy the docker role only -$ ./deploy.sh services SERVICE # deploy SERVICE only +$ ./deploy.sh # deploy everything +$ ./deploy.sh base # deploy the ubuntu_base role only +$ ./deploy.sh services # deploy the docker role only +$ ./deploy.sh services SINGLE_SERVICE # deploy SINGLE_SERVICE Docker service only ``` -`SERVICE` should match one of the filenames in -`roles/docker/tasks/services/`, but without the `.yml` extension +`SINGLE_SERVICE` should match one of the filenames in +`roles/docker/tasks/services/`, but without the `.yml` extension. ## Testing @@ -49,3 +49,16 @@ $ vagrant halt # Power down the VM $ vagrant destroy # Power down and delete the VM ``` +The `vagrant` command-line tool does not support supplying extra +variables to Ansible on runtime, so to be able to deploy only parts of +the Ansible playbook to Vagrant, the `deploy.sh` script can be used with +the `--vagrant` flag. Here are some examples: + +```console +$ ./deploy.sh --vagrant base # deploy the ubuntu_base role only in the Vagrant VM +$ ./deploy.sh --vagrant services SINGLE_SERVICE # deploy SINGLE_SERVICE Docker service only in the Vagrant VM +``` + +Note that the `--vagrant` flag should be the first argument when using +the script. + From bb920407f37ad4b09c6ce39312ab9f271916132a Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 26 Nov 2022 17:18:31 +0100 Subject: [PATCH 03/24] Add depends_on conditions to Mastodon --- roles/docker/tasks/services/mastodon.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/roles/docker/tasks/services/mastodon.yml b/roles/docker/tasks/services/mastodon.yml index baeea09b..92f11a7a 100644 --- a/roles/docker/tasks/services/mastodon.yml +++ b/roles/docker/tasks/services/mastodon.yml @@ -63,8 +63,10 @@ # prettier-ignore test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:3000/health || exit 1'] depends_on: - - db - - redis + db: + condition: "service_healthy" + redis: + condition: "service_healthy" volumes: - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system" environment: @@ -88,8 +90,10 @@ ports: - '127.0.0.1:4000:4000' depends_on: - - db - - redis + db: + condition: "service_healthy" + redis: + condition: "service_healthy" environment: VIRTUAL_HOST: "{{ services.mastodon.domain }}" VIRTUAL_PORT: "4000" @@ -103,8 +107,10 @@ environment: DB_POOL: 32 depends_on: - - db - - redis + db: + condition: "service_healthy" + redis: + condition: "service_healthy" networks: - postfix - external_services @@ -120,4 +126,4 @@ postfix: external: true internal_network: - internal: true \ No newline at end of file + internal: true From a89140ef5157bc3e967fb67036269176551844a4 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 26 Nov 2022 20:09:34 +0100 Subject: [PATCH 04/24] Quality of Life lvl 100 --- deploy.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/deploy.sh b/deploy.sh index 939b92b9..72d00f76 100755 --- a/deploy.sh +++ b/deploy.sh @@ -25,11 +25,13 @@ else echo "Deploying service: $2" $BASE_CMD --tags setup_services --extra-vars "single_service=$2" fi - ;; + ;; "base") $BASE_CMD --tags base_only - ;; + ;; *) - echo "Command \"$1\" not found!" + echo >&2 "Command \"$1\" not found!" + exit 1 + ;; esac fi From 7c7379c42cd5b405844152cd8c70acc554a62842 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 26 Nov 2022 20:28:19 +0100 Subject: [PATCH 05/24] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 114d1a1d..e756adb5 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ $ ./deploy.sh services SINGLE_SERVICE # deploy SINGLE_SERVICE Docker service on ``` `SINGLE_SERVICE` should match one of the filenames in -`roles/docker/tasks/services/`, but without the `.yml` extension. +`roles/docker/tasks/services/**/*`, but without the `.yml` extension. ## Testing From e983499f9b8269bdac6a6d7762b92afb4a81ddfb Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 26 Nov 2022 22:13:51 +0100 Subject: [PATCH 06/24] Use value_name='service' in setup services task --- roles/docker/tasks/services.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/docker/tasks/services.yml b/roles/docker/tasks/services.yml index e183bbf6..c41f5e46 100644 --- a/roles/docker/tasks/services.yml +++ b/roles/docker/tasks/services.yml @@ -4,11 +4,11 @@ name: external_services - name: setup services - include_tasks: "services/{{ item.value.file }}" - loop: "{{ services | dict2items }}" + include_tasks: "services/{{ item.service.file }}" + loop: "{{ services | dict2items(value_name='service') }}" when: single_service is not defined and - item.value.file is defined and - item.value.disabled_in_vagrant is not defined + item.service.file is defined and + item.service.disabled_in_vagrant is not defined - name: setup single service include_tasks: "services/{{ services[single_service].file }}" From f3fd5c7c7498518e551feb625663f0b0c83994b7 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 26 Nov 2022 22:48:15 +0100 Subject: [PATCH 07/24] Shorten Jinja2 filter in postfix.yml --- roles/docker/tasks/services/postfix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/docker/tasks/services/postfix.yml b/roles/docker/tasks/services/postfix.yml index c5656866..09595567 100644 --- a/roles/docker/tasks/services/postfix.yml +++ b/roles/docker/tasks/services/postfix.yml @@ -16,5 +16,5 @@ - name: postfix env: # Get all services which have allowed_sender_domain defined - ALLOWED_SENDER_DOMAINS: "{{ services | dict2items | selectattr('value.allowed_sender_domain', 'defined') | map(attribute='value.domain') | list | join(' ') }}" + ALLOWED_SENDER_DOMAINS: "{{ services | dict2items | selectattr('value.allowed_sender_domain', 'true') | map(attribute='value.domain') | join(' ') }}" HOSTNAME: "smtp.data.coop" # the name the smtp server will identify itself as From c8d603b6aa305f3d8826626af72dc532658628eb Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 26 Nov 2022 22:50:32 +0100 Subject: [PATCH 08/24] Add J2Live to README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index e756adb5..125db88b 100644 --- a/README.md +++ b/README.md @@ -62,3 +62,8 @@ $ ./deploy.sh --vagrant services SINGLE_SERVICE # deploy SINGLE_SERVICE Docker Note that the `--vagrant` flag should be the first argument when using the script. +## Nice tools + +- [J2Live](https://j2live.ttl255.com/): A live Jinja2 parser, nice to + test out filters + From ab1f170790f2f3f1bb3fbe1b57a7e6b15605f6ab Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 26 Nov 2022 23:01:12 +0100 Subject: [PATCH 09/24] Opt out of Mailu statistics, and don't hardcode domains --- roles/docker/tasks/services/postfix.yml | 2 +- roles/docker/templates/mailu.env.j2 | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/docker/tasks/services/postfix.yml b/roles/docker/tasks/services/postfix.yml index 09595567..77dc7279 100644 --- a/roles/docker/tasks/services/postfix.yml +++ b/roles/docker/tasks/services/postfix.yml @@ -17,4 +17,4 @@ env: # Get all services which have allowed_sender_domain defined ALLOWED_SENDER_DOMAINS: "{{ services | dict2items | selectattr('value.allowed_sender_domain', 'true') | map(attribute='value.domain') | join(' ') }}" - HOSTNAME: "smtp.data.coop" # the name the smtp server will identify itself as + HOSTNAME: "smtp.{{ base_domain }}" # the name the smtp server will identify itself as diff --git a/roles/docker/templates/mailu.env.j2 b/roles/docker/templates/mailu.env.j2 index 68c63fb3..aa353e04 100644 --- a/roles/docker/templates/mailu.env.j2 +++ b/roles/docker/templates/mailu.env.j2 @@ -29,10 +29,10 @@ SECRET_KEY={{ mailu_secret_key }} SUBNET={{ services.mailu.subnet }} # Main mail domain -DOMAIN=data.coop +DOMAIN={{ base_domain }} # Hostnames for this server, separated with comas -HOSTNAMES=mail.data.coop +HOSTNAMES=mail.{{ base_domain }} # Postmaster local part (will append the main mail domain) POSTMASTER=admin @@ -44,7 +44,7 @@ TLS_FLAVOR=mail AUTH_RATELIMIT=120/minute;1200/hour # Opt-out of statistics, replace with "True" to opt out -DISABLE_STATISTICS=False +DISABLE_STATISTICS=True ################################### # Optional features @@ -117,10 +117,10 @@ WEB_ADMIN=/admin WEB_WEBMAIL=/webmail # Website name -SITENAME=data.coop +SITENAME={{ base_domain }} # Linked Website URL -WEBSITE=https://mail.data.coop +WEBSITE=https://mail.{{ base_domain }} From a1e8203d5523ca567eb8787b92e9661835215da2 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sat, 26 Nov 2022 23:15:09 +0100 Subject: [PATCH 10/24] Don't hardcode domains --- roles/docker/templates/mailu.env.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/docker/templates/mailu.env.j2 b/roles/docker/templates/mailu.env.j2 index aa353e04..52a48652 100644 --- a/roles/docker/templates/mailu.env.j2 +++ b/roles/docker/templates/mailu.env.j2 @@ -32,7 +32,7 @@ SUBNET={{ services.mailu.subnet }} DOMAIN={{ base_domain }} # Hostnames for this server, separated with comas -HOSTNAMES=mail.{{ base_domain }} +HOSTNAMES={{ services.mailu.domain }} # Postmaster local part (will append the main mail domain) POSTMASTER=admin @@ -120,7 +120,7 @@ WEB_WEBMAIL=/webmail SITENAME={{ base_domain }} # Linked Website URL -WEBSITE=https://mail.{{ base_domain }} +WEBSITE=https://{{ services.mailu.domain }} From e45eb022082a71f23ede56e79681ab7c62c57e85 Mon Sep 17 00:00:00 2001 From: reynir Date: Sun, 27 Nov 2022 14:01:55 +0000 Subject: [PATCH 11/24] Don't hardcode domains (#129) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Reynir Björnsson Reviewed-on: https://git.data.coop/data.coop/ansible/pulls/129 Co-authored-by: reynir Co-committed-by: reynir --- roles/docker/defaults/main.yml | 1 + roles/docker/tasks/services/postfix.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 2a239503..2acddce1 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -6,6 +6,7 @@ services: ### Internal services ### postfix: file: postfix.yml + domain: "smtp.{{ base_domain }}" version: "v3.5.0" nginx_proxy: diff --git a/roles/docker/tasks/services/postfix.yml b/roles/docker/tasks/services/postfix.yml index 77dc7279..1fb67df8 100644 --- a/roles/docker/tasks/services/postfix.yml +++ b/roles/docker/tasks/services/postfix.yml @@ -17,4 +17,4 @@ env: # Get all services which have allowed_sender_domain defined ALLOWED_SENDER_DOMAINS: "{{ services | dict2items | selectattr('value.allowed_sender_domain', 'true') | map(attribute='value.domain') | join(' ') }}" - HOSTNAME: "smtp.{{ base_domain }}" # the name the smtp server will identify itself as + HOSTNAME: "{{ services.postfix.domain }}" # the name the smtp server will identify itself as From 59dae865c57852cb9a8f4c50e33898b5fd245de5 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sun, 27 Nov 2022 16:34:20 +0100 Subject: [PATCH 12/24] Add missing file to codimd --- roles/docker/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 2acddce1..b3ad3ce7 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -101,6 +101,7 @@ services: version: 20221009 codimd: + file: codimd.yml domain: "oldpad.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/codimd" From 93b1ed60aebdc21a7e71119c5f808af980cc7fe2 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sun, 27 Nov 2022 17:20:40 +0100 Subject: [PATCH 13/24] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 125db88b..c2651da3 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,9 @@ $ ./deploy.sh services # deploy the docker role only $ ./deploy.sh services SINGLE_SERVICE # deploy SINGLE_SERVICE Docker service only ``` -`SINGLE_SERVICE` should match one of the filenames in -`roles/docker/tasks/services/**/*`, but without the `.yml` extension. +`SINGLE_SERVICE` should match one of the service names in the `services` +dictionary in `roles/docker/defaults/main.yml` (e.g. `gitea` or +`data_coop_website`). ## Testing From bef767ebd87db52b7de7952d1172c6c019b35e3d Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sun, 27 Nov 2022 17:35:40 +0100 Subject: [PATCH 14/24] Update README.md --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c2651da3..66c06e47 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ also be specified. By default, the script deploys everything. Here is a summary of the options that can be used with the script: -```console +```shell $ ./deploy.sh # deploy everything $ ./deploy.sh base # deploy the ubuntu_base role only $ ./deploy.sh services # deploy the docker role only @@ -42,7 +42,7 @@ this to work properly. Here is a summary of the commands that are available with the `vagrant` command-line tool: -```console +```shell $ vagrant up # Create and provision the VM $ vagrant provision # Re-provision the VM $ vagrant ssh # SSH into the VM @@ -55,7 +55,7 @@ variables to Ansible on runtime, so to be able to deploy only parts of the Ansible playbook to Vagrant, the `deploy.sh` script can be used with the `--vagrant` flag. Here are some examples: -```console +```shell $ ./deploy.sh --vagrant base # deploy the ubuntu_base role only in the Vagrant VM $ ./deploy.sh --vagrant services SINGLE_SERVICE # deploy SINGLE_SERVICE Docker service only in the Vagrant VM ``` @@ -67,4 +67,3 @@ the script. - [J2Live](https://j2live.ttl255.com/): A live Jinja2 parser, nice to test out filters - From 3f036ac0eaadb6e95180b3794ab7309b9a381ced Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Sun, 27 Nov 2022 21:00:47 +0100 Subject: [PATCH 15/24] Revert "Update README.md" This reverts commit bef767ebd87db52b7de7952d1172c6c019b35e3d. --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 66c06e47..c2651da3 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ also be specified. By default, the script deploys everything. Here is a summary of the options that can be used with the script: -```shell +```console $ ./deploy.sh # deploy everything $ ./deploy.sh base # deploy the ubuntu_base role only $ ./deploy.sh services # deploy the docker role only @@ -42,7 +42,7 @@ this to work properly. Here is a summary of the commands that are available with the `vagrant` command-line tool: -```shell +```console $ vagrant up # Create and provision the VM $ vagrant provision # Re-provision the VM $ vagrant ssh # SSH into the VM @@ -55,7 +55,7 @@ variables to Ansible on runtime, so to be able to deploy only parts of the Ansible playbook to Vagrant, the `deploy.sh` script can be used with the `--vagrant` flag. Here are some examples: -```shell +```console $ ./deploy.sh --vagrant base # deploy the ubuntu_base role only in the Vagrant VM $ ./deploy.sh --vagrant services SINGLE_SERVICE # deploy SINGLE_SERVICE Docker service only in the Vagrant VM ``` @@ -67,3 +67,4 @@ the script. - [J2Live](https://j2live.ttl255.com/): A live Jinja2 parser, nice to test out filters + From bd9c134e076f7ddc8c21663af2e9f5d995dd5462 Mon Sep 17 00:00:00 2001 From: reynir Date: Mon, 28 Nov 2022 14:27:49 +0000 Subject: [PATCH 16/24] deploy.sh: print usage message (#130) Co-authored-by: reynir Co-committed-by: reynir --- deploy.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index 72d00f76..9f41a333 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,5 +1,13 @@ #!/bin/sh +usage () { + { + echo "Usage: $0 [--vagrant]" + echo "Usage: $0 [--vagrant] base" + echo "Usage: $0 [--vagrant] services [SERVICE]" + } >&2 +} + BASE_CMD="ansible-playbook playbook.yml --ask-vault-pass" if [ "$1" = "--vagrant" ]; then @@ -30,7 +38,7 @@ else $BASE_CMD --tags base_only ;; *) - echo >&2 "Command \"$1\" not found!" + usage exit 1 ;; esac From 6a29cdc84d566cfd6581ec47177dfd648d18090f Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Mon, 28 Nov 2022 18:20:12 +0100 Subject: [PATCH 17/24] Apply benjaoming's suggestions --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c2651da3..1e67a001 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,15 @@ the association's administrators have access to deploy the services. ## Deploying To deploy the services, the included `deploy.sh` script can be used. The -Ansible playbook uses two custom-made roles (in the `roles/` directory). -The script has options to deploy only one of the roles, which are namely -`ubuntu_base` and `docker`, the latter of which is for deploying the -services themselves using Docker containers. Select services only can -also be specified. By default, the script deploys everything. +Ansible playbook uses two custom-made roles (in the `roles/` directory): + +- `ubuntu_base` - used for configuring the OS itself and installing the + necessary packages +- `docker` - used to deploy our services and websites with Docker + containers + +The script has options to deploy only one of the roles. Select services only +can also be specified. By default, the script deploys everything. Here is a summary of the options that can be used with the script: From 789caed704b192aba8dceb9be11b875a1b563d80 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Mon, 28 Nov 2022 18:56:09 +0100 Subject: [PATCH 18/24] Change wording --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1e67a001..c96782a7 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,13 @@ the association's administrators have access to deploy the services. To deploy the services, the included `deploy.sh` script can be used. The Ansible playbook uses two custom-made roles (in the `roles/` directory): -- `ubuntu_base` - used for configuring the OS itself and installing the +- `ubuntu_base` - used to configure the host itself and install the necessary packages - `docker` - used to deploy our services and websites with Docker containers -The script has options to deploy only one of the roles. Select services only -can also be specified. By default, the script deploys everything. +The script has options to deploy only one of the roles. Select services +only can also be specified. By default, the script deploys everything. Here is a summary of the options that can be used with the script: From 09215e117aee2f2a4cfe1f391cf4719188579333 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Mon, 28 Nov 2022 19:24:49 +0100 Subject: [PATCH 19/24] Add 'Contributing' section --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index c96782a7..d27ba9fd 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,18 @@ $ ./deploy.sh --vagrant services SINGLE_SERVICE # deploy SINGLE_SERVICE Docker Note that the `--vagrant` flag should be the first argument when using the script. +## Contributing + +If you want to contribute, you can fork the repository and submit a pull +request. We use a pre-commit hook for linting the YAML files before +every commit, so please use that. To initialize pre-commit, you need to +have Python and GNU make installed. Then, just run the following shell +command: + +```console +$ make init +``` + ## Nice tools - [J2Live](https://j2live.ttl255.com/): A live Jinja2 parser, nice to From 6cd0eadadeabfee3773415ceb856d4a6ed3e86d6 Mon Sep 17 00:00:00 2001 From: Sam Al-Sapti Date: Mon, 28 Nov 2022 19:31:31 +0100 Subject: [PATCH 20/24] Apply valberg's suggestions --- README.md | 51 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index d27ba9fd..97e3b760 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,18 @@ only can also be specified. By default, the script deploys everything. Here is a summary of the options that can be used with the script: -```console -$ ./deploy.sh # deploy everything -$ ./deploy.sh base # deploy the ubuntu_base role only -$ ./deploy.sh services # deploy the docker role only -$ ./deploy.sh services SINGLE_SERVICE # deploy SINGLE_SERVICE Docker service only +```sh +# deploy everything +./deploy.sh + +# deploy the ubuntu_base role only +./deploy.sh base + +# deploy the docker role only +./deploy.sh services + +# deploy SINGLE_SERVICE Docker service only +./deploy.sh services SINGLE_SERVICE ``` `SINGLE_SERVICE` should match one of the service names in the `services` @@ -46,12 +53,21 @@ this to work properly. Here is a summary of the commands that are available with the `vagrant` command-line tool: -```console -$ vagrant up # Create and provision the VM -$ vagrant provision # Re-provision the VM -$ vagrant ssh # SSH into the VM -$ vagrant halt # Power down the VM -$ vagrant destroy # Power down and delete the VM +```sh +# Create and provision the VM +vagrant up + +# Re-provision the VM +vagrant provision + +# SSH into the VM +vagrant ssh + +# Power down the VM +vagrant halt + +# Power down and delete the VM +vagrant destroy ``` The `vagrant` command-line tool does not support supplying extra @@ -59,9 +75,12 @@ variables to Ansible on runtime, so to be able to deploy only parts of the Ansible playbook to Vagrant, the `deploy.sh` script can be used with the `--vagrant` flag. Here are some examples: -```console -$ ./deploy.sh --vagrant base # deploy the ubuntu_base role only in the Vagrant VM -$ ./deploy.sh --vagrant services SINGLE_SERVICE # deploy SINGLE_SERVICE Docker service only in the Vagrant VM +```sh +# deploy the ubuntu_base role only in the Vagrant VM +./deploy.sh --vagrant base + +# deploy SINGLE_SERVICE Docker service only in the Vagrant VM +./deploy.sh --vagrant services SINGLE_SERVICE ``` Note that the `--vagrant` flag should be the first argument when using @@ -75,8 +94,8 @@ every commit, so please use that. To initialize pre-commit, you need to have Python and GNU make installed. Then, just run the following shell command: -```console -$ make init +```sh +make init ``` ## Nice tools From 759ea93dd3ae3ecf6496309315bff5065bef7824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=AD=C3=B0ir=20Valberg=20Gu=C3=B0mundsson?= Date: Fri, 2 Dec 2022 23:10:15 +0100 Subject: [PATCH 21/24] Mastodon: Split sidekiq queues into different containers. Tune postgresql. Set threads and concurrency on web and streaming. --- .../files/configs/mastodon/postgresql.conf | 20 ++++ roles/docker/tasks/services/mastodon.yml | 95 +++++++++++++++---- 2 files changed, 95 insertions(+), 20 deletions(-) create mode 100644 roles/docker/files/configs/mastodon/postgresql.conf diff --git a/roles/docker/files/configs/mastodon/postgresql.conf b/roles/docker/files/configs/mastodon/postgresql.conf new file mode 100644 index 00000000..c0df75bf --- /dev/null +++ b/roles/docker/files/configs/mastodon/postgresql.conf @@ -0,0 +1,20 @@ +# DB Version: 14 +# OS Type: linux +# DB Type: oltp +# Total Memory (RAM): 16 GB +# Connections num: 300 +# Data Storage: hdd + +listen_addresses = '*' +max_connections = 300 +shared_buffers = 4GB +effective_cache_size = 12GB +maintenance_work_mem = 1GB +checkpoint_completion_target = 0.9 +wal_buffers = 16MB +default_statistics_target = 100 +random_page_cost = 4 +effective_io_concurrency = 2 +work_mem = 6990kB +min_wal_size = 2GB +max_wal_size = 8GB \ No newline at end of file diff --git a/roles/docker/tasks/services/mastodon.yml b/roles/docker/tasks/services/mastodon.yml index 92f11a7a..613b97a2 100644 --- a/roles/docker/tasks/services/mastodon.yml +++ b/roles/docker/tasks/services/mastodon.yml @@ -16,16 +16,40 @@ src: files/configs/mastodon/env_file.j2 dest: "{{ services.mastodon.volume_folder }}/env_file" -- name: upload vhost config for root domain +- name: Upload vhost config for root domain template: src: files/configs/mastodon/vhost-mastodon dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.mastodon.domain }}" +- name: Copy postgresql config + copy: + src: files/configs/mastodon/postgresql.conf + dest: "{{ services.mastodon.volume_folder }}/postgres_data/postgresql.conf" + - name: set up mastodon docker_compose: project_name: mastodon pull: yes definition: + + x-sidekiq: &sidekiq + image: "tootsuite/mastodon:{{ services.mastodon.version }}" + restart: always + env_file: "{{ services.mastodon.volume_folder }}/env_file" + depends_on: + db: + condition: "service_healthy" + redis: + condition: "service_healthy" + networks: + - postfix + - external_services + - internal_network + volumes: + - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system" + healthcheck: + test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"] + version: '3' services: db: @@ -70,6 +94,8 @@ volumes: - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system" environment: + MAX_THREADS: 10 + WEB_CONCURRENCY: 3 VIRTUAL_HOST: "{{ services.mastodon.domain }}" VIRTUAL_PORT: "3000" VIRTUAL_PATH: "/" @@ -95,30 +121,59 @@ redis: condition: "service_healthy" environment: + DB_POOL: 15 VIRTUAL_HOST: "{{ services.mastodon.domain }}" VIRTUAL_PORT: "4000" VIRTUAL_PATH: "/api/v1/streaming" - sidekiq: - image: "tootsuite/mastodon:{{ services.mastodon.version }}" - restart: always - env_file: "{{ services.mastodon.volume_folder }}/env_file" - command: bundle exec sidekiq -c 32 + # sidekiq-default-push-pull: DB_POOL = 25, -c 25 for 25 connections + sidekiq-default-push-pull: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q default -q push -q pull environment: - DB_POOL: 32 - depends_on: - db: - condition: "service_healthy" - redis: - condition: "service_healthy" - networks: - - postfix - - external_services - - internal_network - volumes: - - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system" - healthcheck: - test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"] + DB_POOL: 25 + + # sidekiq-default-pull-push: DB_POOL = 25, -c 25 for 25 connections + sidekiq-default-pull-push: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q default -q pull -q push + environment: + DB_POOL: 25 + + # sidekiq-pull-default-push: DB_POOL = 25, -c 25 for 25 connections + sidekiq-pull-default-push: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q pull -q default -q push + environment: + DB_POOL: 25 + + # sidekiq-push-default-pull: DB_POOL = 25, -c 25 for 25 connections + sidekiq-push-default-pull: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q push -q default -q pull + environment: + DB_POOL: 25 + + # sidekiq-push-scheduler: DB_POOL = 5, -c 5 for 5 connections + sidekiq-push-scheduler: + <<: *sidekiq + command: bundle exec sidekiq -c 5 -q push -q scheduler + environment: + DB_POOL: 5 + + # sidekiq-push-mailers: DB_POOL = 5, -c 5 for 5 connections + sidekiq-push-mailers: + <<: *sidekiq + command: bundle exec sidekiq -c 5 -q push -q mailers + environment: + DB_POOL: 5 + + # sidekiq-push-ingress: DB_POOL = 10, -c 10 for 10 connections + sidekiq-push-ingress: + <<: *sidekiq + command: bundle exec sidekiq -c 10 -q push -q ingress + environment: + DB_POOL: 10 networks: external_services: From c4f391140094f312752b98d7f6611091951a3182 Mon Sep 17 00:00:00 2001 From: Sam A Date: Sun, 4 Dec 2022 21:45:32 +0000 Subject: [PATCH 22/24] Always recreate Mastodon containers (#134) Fixes #133. Co-authored-by: Sam Al-Sapti Reviewed-on: https://git.data.coop/data.coop/ansible/pulls/134 Co-authored-by: Sam A. Co-committed-by: Sam A. --- Vagrantfile | 2 +- deploy.sh | 2 +- playbook.yml | 5 +++-- roles/docker/tasks/services/codimd.yml | 13 +++++-------- roles/docker/tasks/services/mastodon.yml | 13 ++++++++----- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 7d00af1e..1e462762 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,7 +13,7 @@ Vagrant.configure(2) do |config| config.vm.hostname = "datacoop" config.vm.provider :virtualbox do |v| - v.memory = 4096 + v.memory = 8192 end config.vm.provision :ansible do |ansible| diff --git a/deploy.sh b/deploy.sh index 9f41a333..d66caa28 100755 --- a/deploy.sh +++ b/deploy.sh @@ -11,7 +11,7 @@ usage () { BASE_CMD="ansible-playbook playbook.yml --ask-vault-pass" if [ "$1" = "--vagrant" ]; then - BASE_CMD="$BASE_CMD --inventory=vagrant_host" + BASE_CMD="$BASE_CMD --verbose --inventory=vagrant_host" shift fi diff --git a/playbook.yml b/playbook.yml index 1ce47fbb..f2c5a1dd 100644 --- a/playbook.yml +++ b/playbook.yml @@ -3,13 +3,14 @@ gather_facts: true become: true vars: - base_domain: data.coop - letsencrypt_email: admin@data.coop ldap_dn: "dc=data,dc=coop" vagrant: "{{ ansible_virtualization_role == 'guest' }}" letsencrypt_enabled: "{{ not vagrant }}" + base_domain: "{{ 'datacoop.devel' if vagrant else 'data.coop' }}" + letsencrypt_email: "admin@{{ base_domain }}" + smtp_host: "postfix" smtp_port: "587" diff --git a/roles/docker/tasks/services/codimd.yml b/roles/docker/tasks/services/codimd.yml index 1e0e950a..6e13c21a 100644 --- a/roles/docker/tasks/services/codimd.yml +++ b/roles/docker/tasks/services/codimd.yml @@ -1,17 +1,15 @@ --- - - name: codimd network docker_network: name: codimd - name: create codimd volume folders file: - name: "{{ codimd.volume_folder }}/{{ volume }}" + name: "{{ services.codimd.volume_folder }}/{{ volume }}" state: directory loop: - "db" - "codimd/uploads" - loop_control: loop_var: volume @@ -24,7 +22,7 @@ networks: - name: codimd volumes: - - "{{ codimd.volume_folder }}/db:/var/lib/postgresql/data" + - "{{ services.codimd.volume_folder }}/db:/var/lib/postgresql/data" env: POSTGRES_USER: "codimd" POSTGRES_PASSWORD: "{{ postgres_passwords.codimd }}" @@ -39,8 +37,7 @@ - name: ldap - name: external_services volumes: - - "{{ codimd.volume_folder }}/codimd/uploads:/codimd/public/uploads" - + - "{{ services.codimd.volume_folder }}/codimd/uploads:/codimd/public/uploads" env: CMD_DB_URL: "postgres://codimd:{{ postgres_passwords.codimd }}@codimd_db:5432/codimd" CMD_ALLOW_EMAIL_REGISTER: "False" @@ -52,6 +49,6 @@ CMD_LDAP_SEARCHBASE: "dc=data,dc=coop" CMD_LDAP_SEARCHFILTER: "(&(uid={{ '{{username}}' }})(objectClass=inetOrgPerson))" CMD_USECDN: "false" - VIRTUAL_HOST: "{{ codimd.domain }}" - LETSENCRYPT_HOST: "{{ codimd.domain }}" + VIRTUAL_HOST: "{{ services.codimd.domain }}" + LETSENCRYPT_HOST: "{{ services.codimd.domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" diff --git a/roles/docker/tasks/services/mastodon.yml b/roles/docker/tasks/services/mastodon.yml index 613b97a2..472ea950 100644 --- a/roles/docker/tasks/services/mastodon.yml +++ b/roles/docker/tasks/services/mastodon.yml @@ -6,6 +6,7 @@ group: "991" loop: - "postgres_data" + - "postgres_config" - "redis_data" - "mastodon_data" loop_control: @@ -21,17 +22,17 @@ src: files/configs/mastodon/vhost-mastodon dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.mastodon.domain }}" -- name: Copy postgresql config +- name: Copy PostgreSQL config copy: src: files/configs/mastodon/postgresql.conf - dest: "{{ services.mastodon.volume_folder }}/postgres_data/postgresql.conf" + dest: "{{ services.mastodon.volume_folder }}/postgres_config/postgresql.conf" -- name: set up mastodon +- name: Set up Mastodon docker_compose: project_name: mastodon - pull: yes + pull: true + recreate: always definition: - x-sidekiq: &sidekiq image: "tootsuite/mastodon:{{ services.mastodon.version }}" restart: always @@ -62,6 +63,8 @@ test: ['CMD', 'pg_isready', '-U', 'postgres'] volumes: - "{{ services.mastodon.volume_folder }}/postgres_data:/var/lib/postgresql/data" + - "{{ services.mastodon.volume_folder }}/postgres_config:/config:ro" + command: postgres -c config_file=/config/postgresql.conf environment: - 'POSTGRES_HOST_AUTH_METHOD=trust' From 73bf2d41baf4b0308623735b2217498ae45956bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=AD=C3=B0ir=20Valberg=20Gu=C3=B0mundsson?= Date: Sun, 4 Dec 2022 22:55:00 +0100 Subject: [PATCH 23/24] Restart all mastodon containers instead of recreating them. --- roles/docker/tasks/services/mastodon.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/docker/tasks/services/mastodon.yml b/roles/docker/tasks/services/mastodon.yml index 472ea950..eae1546f 100644 --- a/roles/docker/tasks/services/mastodon.yml +++ b/roles/docker/tasks/services/mastodon.yml @@ -31,7 +31,7 @@ docker_compose: project_name: mastodon pull: true - recreate: always + restarted: true definition: x-sidekiq: &sidekiq image: "tootsuite/mastodon:{{ services.mastodon.version }}" From 51c8acc1195ce1aba2391c616ea05a8cc51d6c25 Mon Sep 17 00:00:00 2001 From: reynir Date: Mon, 5 Dec 2022 15:37:18 +0000 Subject: [PATCH 24/24] Add pinafore (#135) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I don't find any official docker images, so I set up a fork of the repo and build it with drone: https://git.data.coop/data.coop/pinafore Co-authored-by: Reynir Björnsson Reviewed-on: https://git.data.coop/data.coop/ansible/pulls/135 Co-authored-by: reynir Co-committed-by: reynir --- roles/docker/defaults/main.yml | 5 +++++ roles/docker/tasks/services/pinafore.yml | 14 ++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 roles/docker/tasks/services/pinafore.yml diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index b3ad3ce7..f5a54e71 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -164,6 +164,11 @@ services: version: a21f92bf74308d66cfcd545d49b81eba0211a222 allowed_sender_domain: true + pinafore: + file: pinafore.yml + domain: "pinafore.{{ base_domain }}" + version: v2.4.0 + membersystem: file: membersystem.yml domain: "member.{{ base_domain }}" diff --git a/roles/docker/tasks/services/pinafore.yml b/roles/docker/tasks/services/pinafore.yml new file mode 100644 index 00000000..a275f3a1 --- /dev/null +++ b/roles/docker/tasks/services/pinafore.yml @@ -0,0 +1,14 @@ +- name: Set up Pinafore + docker_container: + name: pinafore + image: "docker.data.coop/pinafore:{{ services.pinafore.version }}" + restart_policy: unless-stopped + networks: + - name: external_services + env: + VIRTUAL_HOST: "{{ services.pinafore.domain }}" + VIRTUAL_PORT: "4002" + LETSENCRYPT_HOST: "{{ services.pinafore.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + labels: + com.centurylinklabs.watchtower.enable: "true"