Merge branch 'main' into watchtower

README.md

@@ -0,0 +1,105 @@
+# data.coop infrastructure
+
+This repository contains the code used to deploy data.coop's services
+and websites. We use Ansible to encode our infrastructure setup. Only
+the association's administrators have access to deploy the services.
+
+## Deploying
+
+To deploy the services, the included `deploy.sh` script can be used. The
+Ansible playbook uses two custom-made roles (in the `roles/` directory):
+
+- `ubuntu_base` - used to configure the host itself and install the
+  necessary packages
+- `docker` - used to deploy our services and websites with Docker
+  containers
+
+The script has options to deploy only one of the roles. Select services
+only can also be specified. By default, the script deploys everything.
+
+Here is a summary of the options that can be used with the script:
+
+```sh
+# deploy everything
+./deploy.sh
+
+# deploy the ubuntu_base role only
+./deploy.sh base
+
+# deploy the docker role only
+./deploy.sh services
+
+# deploy SINGLE_SERVICE Docker service only
+./deploy.sh services SINGLE_SERVICE
+```
+
+`SINGLE_SERVICE` should match one of the service names in the `services`
+dictionary in `roles/docker/defaults/main.yml` (e.g. `gitea` or
+`data_coop_website`).
+
+## Testing
+
+In order for us to be able to test our setup locally, we use Vagrant to
+deploy the services in a virtual machine. To do this, Vagrant and
+VirtualBox must both be installed on the development machine. Then, the
+services can be deployed locally by using the `vagrant` command-line
+tool. The working directory needs to be the root of the repository for
+this to work properly.
+
+> Note: As our secrets are contained in an Ansible Vault file, only the
+> administrators have the ability to run the deployment in Vagrant.
+> However, one could replace the vault file for testing purposes.
+
+Here is a summary of the commands that are available with the `vagrant`
+command-line tool:
+
+```sh
+# Create and provision the VM
+vagrant up
+
+# Re-provision the VM
+vagrant provision
+
+# SSH into the VM
+vagrant ssh
+
+# Power down the VM
+vagrant halt
+
+# Power down and delete the VM
+vagrant destroy
+```
+
+The `vagrant` command-line tool does not support supplying extra
+variables to Ansible on runtime, so to be able to deploy only parts of
+the Ansible playbook to Vagrant, the `deploy.sh` script can be used with
+the `--vagrant` flag. Here are some examples:
+
+```sh
+# deploy the ubuntu_base role only in the Vagrant VM
+./deploy.sh --vagrant base
+
+# deploy SINGLE_SERVICE Docker service only in the Vagrant VM
+./deploy.sh --vagrant services SINGLE_SERVICE
+```
+
+Note that the `--vagrant` flag should be the first argument when using
+the script.
+
+## Contributing
+
+If you want to contribute, you can fork the repository and submit a pull
+request. We use a pre-commit hook for linting the YAML files before
+every commit, so please use that. To initialize pre-commit, you need to
+have Python and GNU make installed. Then, just run the following shell
+command:
+
+```sh
+make init
+```
+
+## Nice tools
+
+- [J2Live](https://j2live.ttl255.com/): A live Jinja2 parser, nice to
+  test out filters
+

Vagrantfile

@@ -13,7 +13,7 @@ Vagrant.configure(2) do |config|
   config.vm.hostname = "datacoop"
 
   config.vm.provider :virtualbox do |v|
-    v.memory = 4096
+    v.memory = 8192
   end
 
   config.vm.provision :ansible do |ansible|

deploy.sh

@@ -1,9 +1,17 @@
 #!/bin/sh
 
+usage () {
+    {
+        echo "Usage: $0 [--vagrant]"
+        echo "Usage: $0 [--vagrant] base"
+        echo "Usage: $0 [--vagrant] services [SERVICE]"
+    } >&2
+}
+
 BASE_CMD="ansible-playbook playbook.yml --ask-vault-pass"
 
 if [ "$1" = "--vagrant" ]; then
-    BASE_CMD="$BASE_CMD --inventory=vagrant_host"
+    BASE_CMD="$BASE_CMD --verbose --inventory=vagrant_host"
     shift
 fi
 
@@ -25,11 +33,13 @@ else
                 echo "Deploying service: $2"
                 $BASE_CMD --tags setup_services --extra-vars "single_service=$2"
             fi
-	    ;;
+        ;;
         "base")
             $BASE_CMD --tags base_only
-	    ;;
+        ;;
         *)
-            echo "Command \"$1\" not found!"
+            usage
+            exit 1
+        ;;
     esac
 fi

playbook.yml

@@ -3,13 +3,14 @@
   gather_facts: true
   become: true
   vars:
-    base_domain: data.coop
-    letsencrypt_email: admin@data.coop
     ldap_dn: "dc=data,dc=coop"
 
     vagrant: "{{ ansible_virtualization_role == 'guest' }}"
     letsencrypt_enabled: "{{ not vagrant }}"
 
+    base_domain: "{{ 'datacoop.devel' if vagrant else 'data.coop' }}"
+    letsencrypt_email: "admin@{{ base_domain }}"
+
     smtp_host: "postfix"
     smtp_port: "587"
 

roles/docker/defaults/main.yml

@@ -6,6 +6,7 @@ services:
   ### Internal services ###
   postfix:
     file: postfix.yml
+    domain: "smtp.{{ base_domain }}"
     version: "v3.5.1"
 
   nginx_proxy:
@@ -100,6 +101,7 @@ services:
     version: 20221009
 
   codimd:
+    file: codimd.yml
     domain: "oldpad.{{ base_domain }}"
     volume_folder: "{{ volume_root_folder }}/codimd"
 
@@ -162,6 +164,11 @@ services:
     version: a21f92bf74308d66cfcd545d49b81eba0211a222
     allowed_sender_domain: true
 
+  pinafore:
+    file: pinafore.yml
+    domain: "pinafore.{{ base_domain }}"
+    version: v2.4.0
+
   membersystem:
     file: membersystem.yml
     domain: "member.{{ base_domain }}"

roles/docker/files/configs/mastodon/postgresql.conf

@@ -0,0 +1,20 @@
+# DB Version: 14
+# OS Type: linux
+# DB Type: oltp
+# Total Memory (RAM): 16 GB
+# Connections num: 300
+# Data Storage: hdd
+
+listen_addresses = '*'
+max_connections = 300
+shared_buffers = 4GB
+effective_cache_size = 12GB
+maintenance_work_mem = 1GB
+checkpoint_completion_target = 0.9
+wal_buffers = 16MB
+default_statistics_target = 100
+random_page_cost = 4
+effective_io_concurrency = 2
+work_mem = 6990kB
+min_wal_size = 2GB
+max_wal_size = 8GB

roles/docker/tasks/services.yml

@@ -4,11 +4,11 @@
     name: external_services
 
 - name: setup services
-  include_tasks: "services/{{ item.value.file }}"
+  include_tasks: "services/{{ item.service.file }}"
-  loop: "{{ services | dict2items }}"
+  loop: "{{ services | dict2items(value_name='service') }}"
   when: single_service is not defined and
-        item.value.file is defined and
+        item.service.file is defined and
-        item.value.disabled_in_vagrant is not defined
+        item.service.disabled_in_vagrant is not defined
 
 - name: setup single service
   include_tasks: "services/{{ services[single_service].file }}"

roles/docker/tasks/services/codimd.yml

@@ -1,17 +1,15 @@
 ---
-
 - name: codimd network
   docker_network:
     name: codimd
 
 - name: create codimd volume folders
   file:
-    name: "{{ codimd.volume_folder }}/{{ volume }}"
+    name: "{{ services.codimd.volume_folder }}/{{ volume }}"
     state: directory
   loop:
     - "db"
     - "codimd/uploads"
-
   loop_control:
     loop_var: volume
 
@@ -24,7 +22,7 @@
     networks:
       - name: codimd
     volumes:
-      - "{{ codimd.volume_folder }}/db:/var/lib/postgresql/data"
+      - "{{ services.codimd.volume_folder }}/db:/var/lib/postgresql/data"
     env:
       POSTGRES_USER: "codimd"
       POSTGRES_PASSWORD: "{{ postgres_passwords.codimd }}"
@@ -39,8 +37,7 @@
       - name: ldap
       - name: external_services
     volumes:
-      - "{{ codimd.volume_folder }}/codimd/uploads:/codimd/public/uploads"
+      - "{{ services.codimd.volume_folder }}/codimd/uploads:/codimd/public/uploads"
-
     env:
       CMD_DB_URL: "postgres://codimd:{{ postgres_passwords.codimd }}@codimd_db:5432/codimd"
       CMD_ALLOW_EMAIL_REGISTER: "False"
@@ -52,6 +49,6 @@
       CMD_LDAP_SEARCHBASE: "dc=data,dc=coop"
       CMD_LDAP_SEARCHFILTER: "(&(uid={{ '{{username}}' }})(objectClass=inetOrgPerson))"
       CMD_USECDN: "false"
-      VIRTUAL_HOST: "{{ codimd.domain }}"
+      VIRTUAL_HOST: "{{ services.codimd.domain }}"
-      LETSENCRYPT_HOST: "{{ codimd.domain }}"
+      LETSENCRYPT_HOST: "{{ services.codimd.domain }}"
       LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"

roles/docker/tasks/services/mastodon.yml

@@ -6,6 +6,7 @@
     group: "991"
   loop:
     - "postgres_data"
+    - "postgres_config"
     - "redis_data"
     - "mastodon_data"
   loop_control:
@@ -16,16 +17,40 @@
     src: files/configs/mastodon/env_file.j2
     dest: "{{ services.mastodon.volume_folder }}/env_file"
 
-- name: upload vhost config for root domain
+- name: Upload vhost config for root domain
   template:
     src: files/configs/mastodon/vhost-mastodon
     dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.mastodon.domain }}"
 
-- name: set up mastodon
+- name: Copy PostgreSQL config
+  copy:
+    src: files/configs/mastodon/postgresql.conf
+    dest: "{{ services.mastodon.volume_folder }}/postgres_config/postgresql.conf"
+
+- name: Set up Mastodon
   docker_compose:
     project_name: mastodon
-    pull: yes
+    pull: true
+    restarted: true
     definition:
+      x-sidekiq: &sidekiq
+        image: "tootsuite/mastodon:{{ services.mastodon.version }}"
+        restart: always
+        env_file: "{{ services.mastodon.volume_folder }}/env_file"
+        depends_on:
+          db:
+            condition: "service_healthy"
+          redis:
+            condition: "service_healthy"
+        networks:
+          - postfix
+          - external_services
+          - internal_network
+        volumes:
+          - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system"
+        healthcheck:
+          test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"]
+
       version: '3'
       services:
         db:
@@ -38,6 +63,8 @@
             test: ['CMD', 'pg_isready', '-U', 'postgres']
           volumes:
             - "{{ services.mastodon.volume_folder }}/postgres_data:/var/lib/postgresql/data"
+            - "{{ services.mastodon.volume_folder }}/postgres_config:/config:ro"
+          command: postgres -c config_file=/config/postgresql.conf
           environment:
             - 'POSTGRES_HOST_AUTH_METHOD=trust'
 
@@ -63,11 +90,15 @@
             # prettier-ignore
             test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:3000/health || exit 1']
           depends_on:
-            - db
+            db:
-            - redis
+              condition: "service_healthy"
+            redis:
+              condition: "service_healthy"
           volumes:
             - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system"
           environment:
+            MAX_THREADS: 10
+            WEB_CONCURRENCY: 3
             VIRTUAL_HOST: "{{ services.mastodon.domain }}"
             VIRTUAL_PORT: "3000"
             VIRTUAL_PATH: "/"
@@ -88,31 +119,64 @@
           ports:
             - '127.0.0.1:4000:4000'
           depends_on:
-            - db
+            db:
-            - redis
+              condition: "service_healthy"
+            redis:
+              condition: "service_healthy"
           environment:
+            DB_POOL: 15
             VIRTUAL_HOST: "{{ services.mastodon.domain }}"
             VIRTUAL_PORT: "4000"
             VIRTUAL_PATH: "/api/v1/streaming"
 
-        sidekiq:
+        # sidekiq-default-push-pull: DB_POOL = 25, -c 25 for 25 connections
-          image: "tootsuite/mastodon:{{ services.mastodon.version }}"
+        sidekiq-default-push-pull:
-          restart: always
+          <<: *sidekiq
-          env_file: "{{ services.mastodon.volume_folder }}/env_file"
+          command: bundle exec sidekiq -c 25 -q default -q push -q pull
-          command: bundle exec sidekiq -c 32
           environment:
-            DB_POOL: 32
+            DB_POOL: 25
-          depends_on:
+
-            - db
+        # sidekiq-default-pull-push: DB_POOL = 25, -c 25 for 25 connections
-            - redis
+        sidekiq-default-pull-push:
-          networks:
+          <<: *sidekiq
-            - postfix
+          command: bundle exec sidekiq -c 25 -q default -q pull -q push
-            - external_services
+          environment:
-            - internal_network
+            DB_POOL: 25
-          volumes:
+
-            - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system"
+        # sidekiq-pull-default-push: DB_POOL = 25, -c 25 for 25 connections
-          healthcheck:
+        sidekiq-pull-default-push:
-            test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"]
+          <<: *sidekiq
+          command: bundle exec sidekiq -c 25 -q pull -q default -q push
+          environment:
+            DB_POOL: 25
+
+        # sidekiq-push-default-pull: DB_POOL = 25, -c 25 for 25 connections
+        sidekiq-push-default-pull:
+          <<: *sidekiq
+          command: bundle exec sidekiq -c 25 -q push -q default -q pull
+          environment:
+            DB_POOL: 25
+
+        # sidekiq-push-scheduler: DB_POOL = 5, -c 5 for 5 connections
+        sidekiq-push-scheduler:
+          <<: *sidekiq
+          command: bundle exec sidekiq -c 5 -q push -q scheduler
+          environment:
+            DB_POOL: 5
+
+        # sidekiq-push-mailers: DB_POOL = 5, -c 5 for 5 connections
+        sidekiq-push-mailers:
+          <<: *sidekiq
+          command: bundle exec sidekiq -c 5 -q push -q mailers
+          environment:
+            DB_POOL: 5
+
+        # sidekiq-push-ingress: DB_POOL = 10, -c 10 for 10 connections
+        sidekiq-push-ingress:
+          <<: *sidekiq
+          command: bundle exec sidekiq -c 10 -q push -q ingress
+          environment:
+            DB_POOL: 10
 
       networks:
         external_services:
@@ -120,4 +184,4 @@
         postfix:
           external: true
         internal_network:
-          internal: true
+          internal: true

roles/docker/tasks/services/pinafore.yml

@@ -0,0 +1,14 @@
+- name: Set up Pinafore
+  docker_container:
+    name: pinafore
+    image: "docker.data.coop/pinafore:{{ services.pinafore.version }}"
+    restart_policy: unless-stopped
+    networks:
+      - name: external_services
+    env:
+      VIRTUAL_HOST: "{{ services.pinafore.domain }}"
+      VIRTUAL_PORT: "4002"
+      LETSENCRYPT_HOST: "{{ services.pinafore.domain }}"
+      LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"
+    labels:
+      com.centurylinklabs.watchtower.enable: "true"

roles/docker/tasks/services/postfix.yml

@@ -16,5 +16,5 @@
       - name: postfix
     env:
       # Get all services which have allowed_sender_domain defined
-      ALLOWED_SENDER_DOMAINS: "{{ services | dict2items | selectattr('value.allowed_sender_domain', 'defined') | map(attribute='value.domain') | list | join(' ') }}"
+      ALLOWED_SENDER_DOMAINS: "{{ services | dict2items | selectattr('value.allowed_sender_domain', 'true') | map(attribute='value.domain') | join(' ') }}"
-      HOSTNAME: "smtp.data.coop" # the name the smtp server will identify itself as
+      HOSTNAME: "{{ services.postfix.domain }}" # the name the smtp server will identify itself as

roles/docker/templates/mailu.env.j2

@@ -29,10 +29,10 @@ SECRET_KEY={{ mailu_secret_key }}
 SUBNET={{ services.mailu.subnet }}
 
 # Main mail domain
-DOMAIN=data.coop
+DOMAIN={{ base_domain }}
 
 # Hostnames for this server, separated with comas
-HOSTNAMES=mail.data.coop
+HOSTNAMES={{ services.mailu.domain }}
 
 # Postmaster local part (will append the main mail domain)
 POSTMASTER=admin
@@ -44,7 +44,7 @@ TLS_FLAVOR=mail
 AUTH_RATELIMIT=120/minute;1200/hour
 
 # Opt-out of statistics, replace with "True" to opt out
-DISABLE_STATISTICS=False
+DISABLE_STATISTICS=True
 
 ###################################
 # Optional features
@@ -117,10 +117,10 @@ WEB_ADMIN=/admin
 WEB_WEBMAIL=/webmail
 
 # Website name
-SITENAME=data.coop
+SITENAME={{ base_domain }}
 
 # Linked Website URL
-WEBSITE=https://mail.data.coop
+WEBSITE=https://{{ services.mailu.domain }}