diff --git a/README.md b/README.md new file mode 100644 index 00000000..97e3b760 --- /dev/null +++ b/README.md @@ -0,0 +1,105 @@ +# data.coop infrastructure + +This repository contains the code used to deploy data.coop's services +and websites. We use Ansible to encode our infrastructure setup. Only +the association's administrators have access to deploy the services. + +## Deploying + +To deploy the services, the included `deploy.sh` script can be used. The +Ansible playbook uses two custom-made roles (in the `roles/` directory): + +- `ubuntu_base` - used to configure the host itself and install the + necessary packages +- `docker` - used to deploy our services and websites with Docker + containers + +The script has options to deploy only one of the roles. Select services +only can also be specified. By default, the script deploys everything. + +Here is a summary of the options that can be used with the script: + +```sh +# deploy everything +./deploy.sh + +# deploy the ubuntu_base role only +./deploy.sh base + +# deploy the docker role only +./deploy.sh services + +# deploy SINGLE_SERVICE Docker service only +./deploy.sh services SINGLE_SERVICE +``` + +`SINGLE_SERVICE` should match one of the service names in the `services` +dictionary in `roles/docker/defaults/main.yml` (e.g. `gitea` or +`data_coop_website`). + +## Testing + +In order for us to be able to test our setup locally, we use Vagrant to +deploy the services in a virtual machine. To do this, Vagrant and +VirtualBox must both be installed on the development machine. Then, the +services can be deployed locally by using the `vagrant` command-line +tool. The working directory needs to be the root of the repository for +this to work properly. + +> Note: As our secrets are contained in an Ansible Vault file, only the +> administrators have the ability to run the deployment in Vagrant. +> However, one could replace the vault file for testing purposes. + +Here is a summary of the commands that are available with the `vagrant` +command-line tool: + +```sh +# Create and provision the VM +vagrant up + +# Re-provision the VM +vagrant provision + +# SSH into the VM +vagrant ssh + +# Power down the VM +vagrant halt + +# Power down and delete the VM +vagrant destroy +``` + +The `vagrant` command-line tool does not support supplying extra +variables to Ansible on runtime, so to be able to deploy only parts of +the Ansible playbook to Vagrant, the `deploy.sh` script can be used with +the `--vagrant` flag. Here are some examples: + +```sh +# deploy the ubuntu_base role only in the Vagrant VM +./deploy.sh --vagrant base + +# deploy SINGLE_SERVICE Docker service only in the Vagrant VM +./deploy.sh --vagrant services SINGLE_SERVICE +``` + +Note that the `--vagrant` flag should be the first argument when using +the script. + +## Contributing + +If you want to contribute, you can fork the repository and submit a pull +request. We use a pre-commit hook for linting the YAML files before +every commit, so please use that. To initialize pre-commit, you need to +have Python and GNU make installed. Then, just run the following shell +command: + +```sh +make init +``` + +## Nice tools + +- [J2Live](https://j2live.ttl255.com/): A live Jinja2 parser, nice to + test out filters + diff --git a/Vagrantfile b/Vagrantfile index 7d00af1e..1e462762 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,7 +13,7 @@ Vagrant.configure(2) do |config| config.vm.hostname = "datacoop" config.vm.provider :virtualbox do |v| - v.memory = 4096 + v.memory = 8192 end config.vm.provision :ansible do |ansible| diff --git a/deploy.sh b/deploy.sh index 939b92b9..d66caa28 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,9 +1,17 @@ #!/bin/sh +usage () { + { + echo "Usage: $0 [--vagrant]" + echo "Usage: $0 [--vagrant] base" + echo "Usage: $0 [--vagrant] services [SERVICE]" + } >&2 +} + BASE_CMD="ansible-playbook playbook.yml --ask-vault-pass" if [ "$1" = "--vagrant" ]; then - BASE_CMD="$BASE_CMD --inventory=vagrant_host" + BASE_CMD="$BASE_CMD --verbose --inventory=vagrant_host" shift fi @@ -25,11 +33,13 @@ else echo "Deploying service: $2" $BASE_CMD --tags setup_services --extra-vars "single_service=$2" fi - ;; + ;; "base") $BASE_CMD --tags base_only - ;; + ;; *) - echo "Command \"$1\" not found!" + usage + exit 1 + ;; esac fi diff --git a/playbook.yml b/playbook.yml index 1ce47fbb..f2c5a1dd 100644 --- a/playbook.yml +++ b/playbook.yml @@ -3,13 +3,14 @@ gather_facts: true become: true vars: - base_domain: data.coop - letsencrypt_email: admin@data.coop ldap_dn: "dc=data,dc=coop" vagrant: "{{ ansible_virtualization_role == 'guest' }}" letsencrypt_enabled: "{{ not vagrant }}" + base_domain: "{{ 'datacoop.devel' if vagrant else 'data.coop' }}" + letsencrypt_email: "admin@{{ base_domain }}" + smtp_host: "postfix" smtp_port: "587" diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 09759066..8dfc4bdc 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -6,6 +6,7 @@ services: ### Internal services ### postfix: file: postfix.yml + domain: "smtp.{{ base_domain }}" version: "v3.5.1" nginx_proxy: @@ -100,6 +101,7 @@ services: version: 20221009 codimd: + file: codimd.yml domain: "oldpad.{{ base_domain }}" volume_folder: "{{ volume_root_folder }}/codimd" @@ -162,6 +164,11 @@ services: version: a21f92bf74308d66cfcd545d49b81eba0211a222 allowed_sender_domain: true + pinafore: + file: pinafore.yml + domain: "pinafore.{{ base_domain }}" + version: v2.4.0 + membersystem: file: membersystem.yml domain: "member.{{ base_domain }}" diff --git a/roles/docker/files/configs/mastodon/postgresql.conf b/roles/docker/files/configs/mastodon/postgresql.conf new file mode 100644 index 00000000..c0df75bf --- /dev/null +++ b/roles/docker/files/configs/mastodon/postgresql.conf @@ -0,0 +1,20 @@ +# DB Version: 14 +# OS Type: linux +# DB Type: oltp +# Total Memory (RAM): 16 GB +# Connections num: 300 +# Data Storage: hdd + +listen_addresses = '*' +max_connections = 300 +shared_buffers = 4GB +effective_cache_size = 12GB +maintenance_work_mem = 1GB +checkpoint_completion_target = 0.9 +wal_buffers = 16MB +default_statistics_target = 100 +random_page_cost = 4 +effective_io_concurrency = 2 +work_mem = 6990kB +min_wal_size = 2GB +max_wal_size = 8GB \ No newline at end of file diff --git a/roles/docker/tasks/services.yml b/roles/docker/tasks/services.yml index e183bbf6..c41f5e46 100644 --- a/roles/docker/tasks/services.yml +++ b/roles/docker/tasks/services.yml @@ -4,11 +4,11 @@ name: external_services - name: setup services - include_tasks: "services/{{ item.value.file }}" - loop: "{{ services | dict2items }}" + include_tasks: "services/{{ item.service.file }}" + loop: "{{ services | dict2items(value_name='service') }}" when: single_service is not defined and - item.value.file is defined and - item.value.disabled_in_vagrant is not defined + item.service.file is defined and + item.service.disabled_in_vagrant is not defined - name: setup single service include_tasks: "services/{{ services[single_service].file }}" diff --git a/roles/docker/tasks/services/codimd.yml b/roles/docker/tasks/services/codimd.yml index 1e0e950a..6e13c21a 100644 --- a/roles/docker/tasks/services/codimd.yml +++ b/roles/docker/tasks/services/codimd.yml @@ -1,17 +1,15 @@ --- - - name: codimd network docker_network: name: codimd - name: create codimd volume folders file: - name: "{{ codimd.volume_folder }}/{{ volume }}" + name: "{{ services.codimd.volume_folder }}/{{ volume }}" state: directory loop: - "db" - "codimd/uploads" - loop_control: loop_var: volume @@ -24,7 +22,7 @@ networks: - name: codimd volumes: - - "{{ codimd.volume_folder }}/db:/var/lib/postgresql/data" + - "{{ services.codimd.volume_folder }}/db:/var/lib/postgresql/data" env: POSTGRES_USER: "codimd" POSTGRES_PASSWORD: "{{ postgres_passwords.codimd }}" @@ -39,8 +37,7 @@ - name: ldap - name: external_services volumes: - - "{{ codimd.volume_folder }}/codimd/uploads:/codimd/public/uploads" - + - "{{ services.codimd.volume_folder }}/codimd/uploads:/codimd/public/uploads" env: CMD_DB_URL: "postgres://codimd:{{ postgres_passwords.codimd }}@codimd_db:5432/codimd" CMD_ALLOW_EMAIL_REGISTER: "False" @@ -52,6 +49,6 @@ CMD_LDAP_SEARCHBASE: "dc=data,dc=coop" CMD_LDAP_SEARCHFILTER: "(&(uid={{ '{{username}}' }})(objectClass=inetOrgPerson))" CMD_USECDN: "false" - VIRTUAL_HOST: "{{ codimd.domain }}" - LETSENCRYPT_HOST: "{{ codimd.domain }}" + VIRTUAL_HOST: "{{ services.codimd.domain }}" + LETSENCRYPT_HOST: "{{ services.codimd.domain }}" LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" diff --git a/roles/docker/tasks/services/mastodon.yml b/roles/docker/tasks/services/mastodon.yml index baeea09b..eae1546f 100644 --- a/roles/docker/tasks/services/mastodon.yml +++ b/roles/docker/tasks/services/mastodon.yml @@ -6,6 +6,7 @@ group: "991" loop: - "postgres_data" + - "postgres_config" - "redis_data" - "mastodon_data" loop_control: @@ -16,16 +17,40 @@ src: files/configs/mastodon/env_file.j2 dest: "{{ services.mastodon.volume_folder }}/env_file" -- name: upload vhost config for root domain +- name: Upload vhost config for root domain template: src: files/configs/mastodon/vhost-mastodon dest: "{{ services.nginx_proxy.volume_folder }}/vhost/{{ services.mastodon.domain }}" -- name: set up mastodon +- name: Copy PostgreSQL config + copy: + src: files/configs/mastodon/postgresql.conf + dest: "{{ services.mastodon.volume_folder }}/postgres_config/postgresql.conf" + +- name: Set up Mastodon docker_compose: project_name: mastodon - pull: yes + pull: true + restarted: true definition: + x-sidekiq: &sidekiq + image: "tootsuite/mastodon:{{ services.mastodon.version }}" + restart: always + env_file: "{{ services.mastodon.volume_folder }}/env_file" + depends_on: + db: + condition: "service_healthy" + redis: + condition: "service_healthy" + networks: + - postfix + - external_services + - internal_network + volumes: + - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system" + healthcheck: + test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"] + version: '3' services: db: @@ -38,6 +63,8 @@ test: ['CMD', 'pg_isready', '-U', 'postgres'] volumes: - "{{ services.mastodon.volume_folder }}/postgres_data:/var/lib/postgresql/data" + - "{{ services.mastodon.volume_folder }}/postgres_config:/config:ro" + command: postgres -c config_file=/config/postgresql.conf environment: - 'POSTGRES_HOST_AUTH_METHOD=trust' @@ -63,11 +90,15 @@ # prettier-ignore test: ['CMD-SHELL', 'wget -q --spider --proxy=off localhost:3000/health || exit 1'] depends_on: - - db - - redis + db: + condition: "service_healthy" + redis: + condition: "service_healthy" volumes: - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system" environment: + MAX_THREADS: 10 + WEB_CONCURRENCY: 3 VIRTUAL_HOST: "{{ services.mastodon.domain }}" VIRTUAL_PORT: "3000" VIRTUAL_PATH: "/" @@ -88,31 +119,64 @@ ports: - '127.0.0.1:4000:4000' depends_on: - - db - - redis + db: + condition: "service_healthy" + redis: + condition: "service_healthy" environment: + DB_POOL: 15 VIRTUAL_HOST: "{{ services.mastodon.domain }}" VIRTUAL_PORT: "4000" VIRTUAL_PATH: "/api/v1/streaming" - sidekiq: - image: "tootsuite/mastodon:{{ services.mastodon.version }}" - restart: always - env_file: "{{ services.mastodon.volume_folder }}/env_file" - command: bundle exec sidekiq -c 32 + # sidekiq-default-push-pull: DB_POOL = 25, -c 25 for 25 connections + sidekiq-default-push-pull: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q default -q push -q pull environment: - DB_POOL: 32 - depends_on: - - db - - redis - networks: - - postfix - - external_services - - internal_network - volumes: - - "{{ services.mastodon.volume_folder }}/mastodon_data:/mastodon/public/system" - healthcheck: - test: ['CMD-SHELL', "ps aux | grep '[s]idekiq\ 6' || false"] + DB_POOL: 25 + + # sidekiq-default-pull-push: DB_POOL = 25, -c 25 for 25 connections + sidekiq-default-pull-push: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q default -q pull -q push + environment: + DB_POOL: 25 + + # sidekiq-pull-default-push: DB_POOL = 25, -c 25 for 25 connections + sidekiq-pull-default-push: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q pull -q default -q push + environment: + DB_POOL: 25 + + # sidekiq-push-default-pull: DB_POOL = 25, -c 25 for 25 connections + sidekiq-push-default-pull: + <<: *sidekiq + command: bundle exec sidekiq -c 25 -q push -q default -q pull + environment: + DB_POOL: 25 + + # sidekiq-push-scheduler: DB_POOL = 5, -c 5 for 5 connections + sidekiq-push-scheduler: + <<: *sidekiq + command: bundle exec sidekiq -c 5 -q push -q scheduler + environment: + DB_POOL: 5 + + # sidekiq-push-mailers: DB_POOL = 5, -c 5 for 5 connections + sidekiq-push-mailers: + <<: *sidekiq + command: bundle exec sidekiq -c 5 -q push -q mailers + environment: + DB_POOL: 5 + + # sidekiq-push-ingress: DB_POOL = 10, -c 10 for 10 connections + sidekiq-push-ingress: + <<: *sidekiq + command: bundle exec sidekiq -c 10 -q push -q ingress + environment: + DB_POOL: 10 networks: external_services: @@ -120,4 +184,4 @@ postfix: external: true internal_network: - internal: true \ No newline at end of file + internal: true diff --git a/roles/docker/tasks/services/pinafore.yml b/roles/docker/tasks/services/pinafore.yml new file mode 100644 index 00000000..a275f3a1 --- /dev/null +++ b/roles/docker/tasks/services/pinafore.yml @@ -0,0 +1,14 @@ +- name: Set up Pinafore + docker_container: + name: pinafore + image: "docker.data.coop/pinafore:{{ services.pinafore.version }}" + restart_policy: unless-stopped + networks: + - name: external_services + env: + VIRTUAL_HOST: "{{ services.pinafore.domain }}" + VIRTUAL_PORT: "4002" + LETSENCRYPT_HOST: "{{ services.pinafore.domain }}" + LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}" + labels: + com.centurylinklabs.watchtower.enable: "true" diff --git a/roles/docker/tasks/services/postfix.yml b/roles/docker/tasks/services/postfix.yml index c5656866..1fb67df8 100644 --- a/roles/docker/tasks/services/postfix.yml +++ b/roles/docker/tasks/services/postfix.yml @@ -16,5 +16,5 @@ - name: postfix env: # Get all services which have allowed_sender_domain defined - ALLOWED_SENDER_DOMAINS: "{{ services | dict2items | selectattr('value.allowed_sender_domain', 'defined') | map(attribute='value.domain') | list | join(' ') }}" - HOSTNAME: "smtp.data.coop" # the name the smtp server will identify itself as + ALLOWED_SENDER_DOMAINS: "{{ services | dict2items | selectattr('value.allowed_sender_domain', 'true') | map(attribute='value.domain') | join(' ') }}" + HOSTNAME: "{{ services.postfix.domain }}" # the name the smtp server will identify itself as diff --git a/roles/docker/templates/mailu.env.j2 b/roles/docker/templates/mailu.env.j2 index 68c63fb3..52a48652 100644 --- a/roles/docker/templates/mailu.env.j2 +++ b/roles/docker/templates/mailu.env.j2 @@ -29,10 +29,10 @@ SECRET_KEY={{ mailu_secret_key }} SUBNET={{ services.mailu.subnet }} # Main mail domain -DOMAIN=data.coop +DOMAIN={{ base_domain }} # Hostnames for this server, separated with comas -HOSTNAMES=mail.data.coop +HOSTNAMES={{ services.mailu.domain }} # Postmaster local part (will append the main mail domain) POSTMASTER=admin @@ -44,7 +44,7 @@ TLS_FLAVOR=mail AUTH_RATELIMIT=120/minute;1200/hour # Opt-out of statistics, replace with "True" to opt out -DISABLE_STATISTICS=False +DISABLE_STATISTICS=True ################################### # Optional features @@ -117,10 +117,10 @@ WEB_ADMIN=/admin WEB_WEBMAIL=/webmail # Website name -SITENAME=data.coop +SITENAME={{ base_domain }} # Linked Website URL -WEBSITE=https://mail.data.coop +WEBSITE=https://{{ services.mailu.domain }}