Add required pip packages.

Reviewed-on: data.coop/ansible#54

group_vars/all/secrets.yml

@@ -1,67 +1,73 @@
 $ANSIBLE_VAULT;1.1;AES256
-36303830393964636165353932336334643761653732643036303563313439623836626231313739
-6239326131333263383438613161353435393036663162310a646237306461633337383838306139
-37303434653266303866643162306433636532333131366132366431303766306232653837653463
-6139336230306131660a636165373732313930396632336532366239303766393937386135306235
-62633837363138323365646235333137646363626430633339376636333838663738316535396161
-66653931666565353337623336636338383035333133643639323739393565396434373734323235
-34306132666430376537643431343362383734343861633332373838383837316266336131353231
-31356665313333343139303337633530643534376338343631396431663937616535306136343266
-36323761653933386231386339666333306331326161306536616238623836306335373930386539
-32326331366435386132393731633664333862656130636632656335353939656638613863633637
-64366662623531333933613566356632383061656165323839623732306331653133393463373465
-61363036613234353962353135366361363537313138626337376138383335633865663237303464
-37373032353066323532313364333863663739613631633230323339346634663733326430343263
-64636331616434343331333065633936313265393739353937396165363337366263393131316661
-34643263343263306235326562663065343438363338326138653066326432646531643764396161
-34343834643134363662373934346463383263636436346433303231656535626664626530326131
-39303231646537316136653932306135303931333762343434623636623436383561353131303163
-63373833656430336365316433663538643634303962623538663362383536613136366230396533
-37363963613532666434653863623030653434303463616638653135666433613030656335396238
-37356632376366383463343736303265326436383435363335333838313264333134653932616465
-31363832656563383035306537376532343934373933306232333631363132616665663433666133
-36643131386134303362373264356437333236663232373035363765303936313162353930353033
-38303864643237323437653463313238613036386461303066353465336230303632613432353661
-36313761303234303363346432666638323865663764623862623165633234363437653964343830
-62656365363538303035326239663264646163366332306332383262356130356436366661383234
-66643832656661623132303562333432656365323836363133373932333732323133383832623939
-37333632623933656336316161376565653363633330646239666266313336306563366462376531
-34623636646439366438316530386334313233613963643634366135373661373138316466313739
-64623835613166616165383463323137303431303035363733366563623934366439323666653633
-65663532343939343334616662366633353231636135346564643034666331616666316561386664
-30636535633432336438663462356232383737393031383431666664303563623932666630653233
-65323932303638653562663734386637323964333331356262346463333065633565386363393064
-37646563366437646330353765623236306561646433636534666438323036373132363036626438
-61393866626236316238326236656164643864343734353438313466633136343931376634366261
-64353631363665366237356566323532306563333937343665393166336535383636303930346362
-38333733306430636436623661653066313936626362373464396138343439376430626162316466
-66623837353763333864663562653166316665363537636537626234666230636665396438313832
-61613330396134666437306162386335613466383231633034393639366533363137613964383135
-35356136393962353262313635343031323638326163336433303066393939646536653033343530
-38303935663038303639616661346664666435393266653663373435363433386461633133343964
-65633532656566613638333266393131653863336563386262643630633164346139383436326463
-62316131363335376336366332323564383036333139386462353933316461346363626465396631
-39616164316463323162616235653365363737373736646331303436303037333536633466303533
-32663335386332663334653734303136333066636234643537393931626333303735386238373763
-33323333646531633964613633356539636538306163303938373638623230396661663538653463
-35396434303634316631376138336530393134353235383637326466306363313462323330623535
-33353335306364323335316336643233386561306134636461336537353838373830373031616636
-62333335616639353436303336653965303132316439316362303931373465376264396139346631
-61393064326433366337613266306263336163393731633165303536636132636638656339373634
-32313637666334313935613564316331396437343335313336666137366133376666306138383163
-62383131616339393534613231616134383562346163343430323261336531306332363736396663
-39643134633838393566643237656663313161326631316437663463653566343238343736656566
-35663231623533323261326162356137646436643766646439663538306661643861356565666434
-38353131363863666439316666353965663531323662376235636533363262353131323166326335
-61656432666130313762376334656434333532636165313834333634646465346234653662393735
-35643138623832646633666366616436323730383066323637316635336134356461353666636164
-30633833313863656237653762373965376638393232376532666361396363303030383138616631
-32633739626565383161343065646232636464373538626330636466316337663739376638386337
-32383763313064666464623137373036336562633734313663313463386430313838636165313962
-31383466363466396337333837626462303434353739343338313137633736313664346239666237
-62313462366136313037646264663533323738656138353235646136383334613035383236383865
-38666432313761653234636363396361643139353730323237616663323433363463373666613765
-31656430326138373434356130626465643733313065366562313566326633653832316265636538
-35396164313932356432306639653461343063643666656265333033643863303637313031653134
-39636436636333343030333762376463303538626665343434633236353236373765643231383933
-3130336364366434306366623062613130616230366633333536
+34376131343263336262656463373830643861336631626539643663333239313831626236306530
+3335623130653432636133356363656465346366303062370a346130326536366638633536613161
+62623334363537636634373231353564396362343330623562383939373538633066616565306235
+3332323863353334640a396462313862366362366535383737376333383361303065383937396530
+38326331396333396263363762346331356431623532343938613834663830393337646666336435
+66356439333434356165613030306138666163653934386233663362646534303737323030636234
+31616132613830363136666639386462363135656432373236393034316664363637663336366435
+64373238633064623735666335636231656231666434383066313336303137333663333031363638
+31643733336535383338376631656439633962653262356335383638373764353530643234303935
+62383930393634613530643739643335616164633038326638356135623561326165376530363461
+37373032393331653261373538633065333662393366666161396638383932393331623766343035
+30333335663039323931306162313538373334393335306132626336643363323839633761383063
+35343632363837383132656437303138303764316439343663303964396463363638336533653930
+39303236353766373131623363653835666439333164366563346164626464633633363163323864
+65363961393237666433623565343832306663323862666333343665376135646132363466616364
+62356331666432336661343762333961333634396466333465633164326239386266643230393566
+36376461373631636630303861313538333834646461663539623738636636626537656438646431
+38383436393238363038313563633634396335346138626666366437333433383039363332623639
+31396165346431333838393164616339656634346561313737306562343562323030613266633263
+61333263653938653034356631333664323630306461346532626533363665363133376232316132
+61346431383230656134373630653264363430383561313866363235333435633966386266653964
+33363534343634343232373036633330613038303437333033313061313932373739343663303931
+34333833386638353436653831623835323032303134366635613735643662636336616464313330
+36633335613630663233326166633565386238656236633261396235363165656333333235643137
+32623461663562313533333835396233383330613661646431646365343430626662326638653330
+38646232386263356566373561353130616539346630613363313163363262356264653233313862
+34386331363236386534353534616531643264613764343362646366393435383332653664353363
+62333935363132373434613038353632643336633136656266316466373734646234636638316265
+62646261396465623561633964313065626361316630353965616233356565343834656563353830
+38346361336237646331366632633130613330336637326163663463386233643734356165666431
+61396263656237333138356231306437653337656133663031303031616437633564613733316264
+63633930353033636235653961393330326635626666626235336334653762373262633739356263
+32323532333463653937386430663437303238313130643435353739393639303033343865323736
+35366139643166626364373663333266376133636433653261316566366630396666336637326664
+30343039633133626435363364346666613732666335313865326234366136366130616334396338
+61663461623432303930623261336464643830303631396430363637383838616432356634303332
+61346536313035376139313638393737393136643366366364363862383335353533313534366534
+61356136366465373530393835613834366665653334376539303462336138646438653039306261
+36613736323566636634666331396463623439323063356232306631616135623231336439303739
+65393837653837336235396532323465656463636238643038383363616633383866333633663831
+61363634356634636265663837306232303362313564323463303363323931396438646337363161
+61313033343532336563393632373830326631616462616263346363636566663966396330386464
+62613039323065343838653439303333396536366537313335353834613338623961646235633764
+30333032323333663530613736313765343364363433366436666134623663653336386632333437
+64386639636237333138323431333234316432366236613530376234636438356531636630396431
+63643833366136363962346632616161363565336163313764383030303337346565613939383563
+65306137633965326534356666346238363137323233336561643333386265613863396338383134
+35363135303232376364306234323435356330333061613663326563343533636165356537336536
+61656131343966346365396133666662393930663237643134383963303766306534633034356335
+37633732393266633965616330643061616664336430643630633033326335643438373737653164
+34633737303533666335306466306330343233326531343065666138633166383664333130653864
+37623730333532633936316461333066313065316664383934343731616430366135346138663531
+33353134333934376663336366663036383630393031303731653332373335333131633136616537
+33666266373439346633373735643339653333626237623530346436306438396332613863346264
+30346431393735326566393633626535383538343866653262653330366330623930646631663961
+38656138313932623131613537376139666137653063313339666333313364343738306439656264
+32346533646465376135376531383132396337653966393133316436616563613135353863653064
+31373466616135393036333037623164346539323463333037613030386666396363353364396439
+39616536646638623739623834363662643566393430623632646434336162316362653434343337
+36623334303866343533623538663531303366343136636631376334653636313264376330313836
+66333131343062373138663330313633623166303337306466313362343034316364666666373965
+36373933343338646333373962623034353631623535306230346663373530346438386334303536
+62366666646263303764303330353835633163363666303133333730343263613039346162356532
+37323133613037313430366238313261633165643563666239623730653164666264633964626461
+31323536623335636333393338333166346336323132373466396432613133613933356232373532
+30653564323031636231343232646165653163393663663731313033323763663965356466366562
+33303830656238653164646161366265636566393436323135356630393033316337363361306363
+30393766636237336466353431616130653961326431323161313234333963643032393061303265
+33396664336535353164643462303636616265306338333634376664323837303238623638313266
+37643861343034646532626164353238373031633861623663316638333039643036353932323962
+39616136653639313232326362663834333363633562646563393561396464383765616230333230
+39663939326332333362

group_vars/all/secrets.yml.contents

@@ -1,4 +1,5 @@
 # These are the variables contained in secrets.yml
+# Secrets are usually 32 characters or more, matching [a-Z0-9]
 
 postgres_passwords:
   fider: xxx
@@ -25,3 +26,7 @@ drone_secrets:
   oauth_client_id: xxx
   oauth_client_secret: xxx
   rpc_shared_secret: xxx
+
+restic_secrets:
+  user_secret: xxx
+  encryption_secret: xxx

group_vars/all/vars.yml

@@ -21,11 +21,4 @@ users:
     groups:
       - sudo
 
-  deni:
-    comment: Denis Smajlović
-    key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC0pB42dtqruXr2Ha8Rdp3QoSrMOLU5cbLMfuNTNmgwPZmprhGGRTAsz8E0aG+5HCFBmW6JxiTGyHGdIZrhYWpKYshxxn4zODfitP7IzDgvg9Pz0Ugw/c5i5eKjkVfw9xLZKjO/H3Ql8R+wFMawpdw7j+Q9G+J3eIidEI6TmSvJVWijBxpLI4qrLceL084qg93XpJENYBOcvx9fYQQ+Es0jo4hmPuHWq5VIkIoIfvVvdhwjlaBsqv2je5BNx8uTMVIyV34ZHpJc95wJ6MOcqpQunW2bR4mGc9FLh67eP2ba9nDoQ0pnnlWGqAIKx5P1ELIg7RM3HhuwfRVh4DxCfjXpMz3l1gr1dA1wVD9bOtzbPWLjo7LNkRcT/loba/jpznTNIEv+kawmm/H0aTZZdlofkGHuX0iOpLV1c1tCDMH1s/MCqwosSRihrXhSOozsABvaaZpJiHcpE4DcNr+xJhe1XrHMqR30KU0r7ulIP8JdhzMNEsq3HxZQh9s8XLIUukrpcpOgJYyGuQK3kG6li5kYbmVqjLDQ1xHi/4r/4TXjJYpsFCZ71SURLzxoF1nauHpBLKcHI3MPFvqr27SQcM92fO14wKDAmAtHINwzMQjlda6B0jFNC+2xUfHrH7yIhwSktytq5qbhiFKK71DSLiMKFvOqoTVWRckO2+Zw1HZB+Q== deni@deni.dk
-    password: $6$ooKtdqASOxYJN2CJ$ziMrGG/qIOeyiTGjx.hPYjVBlHpzM9YY6qCJB9/L8aR33feUvXxZNBCgME93ZhbwDBjblS/tP796LsCF4i17D.
-    groups:
-      - sudo
-
 volume_root_folder: "/docker-volumes"

roles/docker/defaults/main.yml

@@ -19,6 +19,7 @@ gitea:
 
 passit:
   domain: "passit.{{ base_domain }}"
+  volume_folder: "{{ volume_root_folder }}/passit"
 
 fider:
   domain: "feedback.{{ base_domain }}"
@@ -28,7 +29,9 @@ matrix:
   volume_folder: "{{ volume_root_folder }}/matrix"
 
 riot:
-  domain: "riot.{{ base_domain }},element.{{ base_domain }}"
+  domains: 
+  - "riot.{{ base_domain }}"
+  - "element.{{ base_domain }}"
   volume_folder: "{{ volume_root_folder }}/riot"
 
 privatebin:
@@ -49,16 +52,25 @@ docker_registry:
   password: "{{ docker_password }}"
 
 data_coop_website:
-  domain: "{{ base_domain }},www.{{ base_domain }}"
+  domains: 
+  - "{{ base_domain }}"
+  - "www.{{ base_domain }}"
 
 cryptohagen_website:
-  domain: "cryptohagen.dk,www.cryptohagen.dk"
+  domains: 
+  - "cryptohagen.dk"
+  - "www.cryptohagen.dk"
 
 ulovliglogning_website:
-  domain: "ulovliglogning.dk,www.ulovliglogning.dk,ulovlig-logning.dk"
+  domains: 
+  - "ulovliglogning.dk"
+  - "www.ulovliglogning.dk"
+  - "ulovlig-logning.dk"
 
 cryptoaarhus_website:
-  domain: "cryptoaarhus.dk,www.cryptoaarhus.dk"
+  domains: 
+  - "cryptoaarhus.dk"
+  - "www.cryptoaarhus.dk"
 
 drone:
   domain: "drone.{{ base_domain }}"
@@ -78,4 +90,3 @@ portainer:
 ttrss:
   domain: rss.{{ base_domain }}
   volume_folder: "{{ volume_root_folder }}/tt-rss"
-

roles/docker/tasks/services/mailu.yml

@@ -38,7 +38,7 @@
     force: yes
 
 - name: run mail server containers
-  docker_service:
+  docker_compose:
     project_name: mail_server
     pull: yes
     definition:

roles/docker/tasks/services/matrix_riot.yml

@@ -49,7 +49,7 @@
 - name: upload vhost config for riot domain
   template:
     src: files/configs/matrix/vhost-riot
-    dest: "{{ nginx.volume_folder }}/vhost/{{ riot.domain }}"
+    dest: "{{ nginx.volume_folder }}/vhost/{{ riot.domains[0] }}"
 
 - name: upload homeserver.yaml
   template:
@@ -62,7 +62,7 @@
     dest: "{{ matrix.volume_folder }}/data/matrix.data.coop.log.config"
 
 - name: set up matrix and riot
-  docker_service:
+  docker_compose:
     project_name: matrix
     pull: yes
     definition:
@@ -112,9 +112,9 @@
           volumes:
             - "{{ riot.volume_folder }}/data:/data"
           environment:
-            VIRTUAL_HOST: "{{ riot.domain }}"
+            VIRTUAL_HOST: "{{ riot.domains|join(',') }}"
             VIRTUAL_PORT: "8080"
-            LETSENCRYPT_HOST: "{{ riot.domain }}"
+            LETSENCRYPT_HOST: "{{ riot.domains|join(',') }}"
             LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"
 
       networks:

roles/docker/tasks/services/passit.yml

@@ -1,45 +1,47 @@
 ---
 
-- name: passit network
-  docker_network:
-    name: passit
+- name: setup passit containers
+  docker_compose:
+    project_name: "passit"
+    pull: "yes"
+    definition:
+      version: "3.6"
+      services:
 
-- name: passit database volume
-  docker_volume:
-    name: passit_db
+        passit_db:
+          image: "postgres:10"
+          restart: "always"
+          networks:
+            - "passit"
+          volumes:
+            - "{{ passit.volume_folder }}/data:/var/lib/postgresql/data"
+          environment:
+            POSTGRES_USER: "passit"
+            POSTGRES_PASSWORD: "{{ postgres_passwords.passit }}"
 
-- name: passit database container
-  docker_container:
-    name: passit_db
-    image: postgres:10
-    state: started
-    restart_policy: always
-    networks:
-      - name: passit
-    volumes:
-      - passit_db:/var/lib/postgresql/data
-    env:
-      POSTGRES_USER: passit
-      POSTGRES_PASSWORD: "{{ postgres_passwords.passit }}"
+        passit_app:
+          image: "passit/passit:stable"
+          command: "bin/start.sh"
+          restart: "always"
+          networks:
+            - "passit"
+            - "postfix"
+            - "external_services"
+          environment:
+            DATABASE_URL: "postgres://passit:{{ postgres_passwords.passit }}@passit_db:5432/passit"
+            SECRET_KEY: "{{ passit_secret_key }}"
+            IS_DEBUG: 'False'
+            EMAIL_URL: "smtp://noop@{{ smtp_host }}:{{ smtp_port }}"
+            DEFAULT_FROM_EMAIL: "noreply@{{ passit.domain }}"
+            EMAIL_CONFIRMATION_HOST: "https://{{ passit.domain }}"
 
-- name: passit app container
-  docker_container:
-    name: passit
-    image: passit/passit:stable
-    command: bin/start.sh
-    restart_policy: always
-    networks:
-      - name: passit
-      - name: postfix
-      - name: external_services
-    env:
-      DATABASE_URL: "postgres://passit:{{ postgres_passwords.passit }}@passit_db:5432/passit"
-      SECRET_KEY: "{{ passit_secret_key }}"
-      IS_DEBUG: 'False'
-      EMAIL_URL: smtp://noop@{{ smtp_host }}:{{ smtp_port }}
-      DEFAULT_FROM_EMAIL: "noreply@{{ passit.domain }}"
-      EMAIL_CONFIRMATION_HOST: "https://{{ passit.domain }}"
+            VIRTUAL_HOST: "{{ passit.domain }}"
+            LETSENCRYPT_HOST: "{{ passit.domain }}"
+            LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"
 
-      VIRTUAL_HOST: "{{ passit.domain }}"
-      LETSENCRYPT_HOST: "{{ passit.domain }}"
-      LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"
+      networks:
+        passit:
+        postfix:
+          external: true
+        external_services:
+          external: true

roles/docker/tasks/services/portainer.yml

@@ -8,7 +8,7 @@
 - name: run portainer
   docker_container:
     name: portainer
-    image: portainer/portainer:1.23.1
+    image: portainer/portainer-ce:2.0.1
     restart_policy: always
     networks:
       - name: external_services
@@ -19,5 +19,6 @@
       - 9001:9000
     env:
       VIRTUAL_HOST: "{{ portainer.domain }}"
+      VIRTUAL_PORT: "9000"
       LETSENCRYPT_HOST: "{{ portainer.domain }}"
       LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"

roles/docker/tasks/services/restic-backup.yml

@@ -0,0 +1,38 @@
+---
+- name: setup restic backup
+  docker_compose:
+    project_name: restic_backup
+    pull: yes
+    definition:
+      version: '3.6'
+      services:
+        restic-backup:
+          image: mazzolino/restic
+          restart: always
+          environment:
+            RUN_ON_STARTUP: "true"
+            BACKUP_CRON: "0 30 3 * * *"
+            RESTIC_REPOSITORY: "rest:https://datacoop:{{ restic_secrets.user_secret }}@restic.graffen.io/datacoop-hevonen" 
+            RESTIC_PASSWORD: "{{ restic_secrets.encryption_secret }}"
+            RESTIC_BACKUP_SOURCES: "/mnt/volumes"
+            RESTIC_BACKUP_ARGS: >-
+              --tag datacoop-volumes
+              --exclude='*.tmp'
+              --verbose
+            RESTIC_FORGET_ARGS: >-
+              --keep-last 10
+              --keep-daily 7
+              --keep-weekly 5
+              --keep-monthly 12
+            TZ: Europe/Copenhagen
+          volumes:
+            - /docker-volumes:/mnt/volumes:ro
+        
+        restic-prune:
+          image: "mazzolino/restic"
+          environment:
+            RUN_ON_STARTUP: "true"
+            PRUNE_CRON: "0 0 4 * * *"
+            RESTIC_REPOSITORY: "rest:https://datacoop:{{ restic_secrets.user_secret }}@restic.graffen.io/datacoop-hevonen"
+            RESTIC_PASSWORD: "{{ restic_secrets.encryption_secret }}"
+            TZ: Europe/copenhagen

roles/docker/tasks/services/tt-rss.yml

@@ -10,7 +10,7 @@
     loop_var: volume
 
 - name: "set up tt-rss"
-  docker_service:
+  docker_compose:
     project_name: "tt-rss"
     pull: yes
     definition:

roles/docker/tasks/services/ulovliglogning-dk.yml

@@ -6,8 +6,8 @@
     networks:
       - name: external_services
     env:
-      VIRTUAL_HOST: "{{ ulovliglogning_website.domain }}"
-      LETSENCRYPT_HOST: "{{ ulovliglogning_website.domain }}"
+      VIRTUAL_HOST: "{{ ulovliglogning_website.domains|join(',') }}"
+      LETSENCRYPT_HOST: "{{ ulovliglogning_website.domains|join(',') }}"
       LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"
     labels:
       com.ouroboros.enable: "true"

roles/docker/tasks/services/websites.yml

@@ -8,8 +8,8 @@
     networks:
       - name: external_services
     env:
-      VIRTUAL_HOST : "{{ data_coop_website.domain }}"
-      LETSENCRYPT_HOST: "{{ data_coop_website.domain }}"
+      VIRTUAL_HOST : "{{ data_coop_website.domains|join(',') }}"
+      LETSENCRYPT_HOST: "{{ data_coop_website.domains|join(',') }}"
       LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"
     labels:
       com.ouroboros.enable: "true"
@@ -22,8 +22,8 @@
     networks:
       - name: external_services
     env:
-      VIRTUAL_HOST : "new.{{ data_coop_website.domain }}"
-      LETSENCRYPT_HOST: "new.{{ data_coop_website.domain }}"
+      VIRTUAL_HOST : "new.{{ data_coop_website.domains|join(',') }}"
+      LETSENCRYPT_HOST: "new.{{ data_coop_website.domains|join(',') }}"
       LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"
     labels:
       com.ouroboros.enable: "true"
@@ -36,8 +36,8 @@
     networks:
       - name: external_services
     env:
-      VIRTUAL_HOST : "{{ cryptohagen_website.domain }}"
-      LETSENCRYPT_HOST: "{{ cryptohagen_website.domain }}"
+      VIRTUAL_HOST : "{{ cryptohagen_website.domains|join(',') }}"
+      LETSENCRYPT_HOST: "{{ cryptohagen_website.domains|join(',') }}"
       LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"
     labels:
       com.ouroboros.enable: "true"  
@@ -50,8 +50,8 @@
     networks:
       - name: external_services
     env:
-      VIRTUAL_HOST : "{{ cryptoaarhus_website.domain }}"
-      LETSENCRYPT_HOST: "{{ cryptoaarhus_website.domain }}"
+      VIRTUAL_HOST : "{{ cryptoaarhus_website.domains|join(',') }}"
+      LETSENCRYPT_HOST: "{{ cryptoaarhus_website.domains|join(',') }}"
       LETSENCRYPT_EMAIL: "{{ letsencrypt_email }}"
     labels:
       com.ouroboros.enable: "true"

roles/ubuntu_base/tasks/base.yml

@@ -1,5 +1,5 @@
 ---
-- name: Install necessary packages
+- name: Install necessary packages via apt
   apt:
     name: "{{ packages }}"
   vars:
@@ -8,4 +8,11 @@
     - python3-pip
     - apparmor
     - haveged
-    
+
+- name: Install necessary packages via pip
+  pip:
+    name: "{{ packages }}"
+  vars:
+    packages:
+      - docker
+      - docker-compose