From 2c99799d4db5875391285dfbe686b0ee1c5253a1 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Tue, 23 Jul 2024 22:48:24 +0000 Subject: [PATCH 1/4] Shorten and document Docker tweaks for Python and pip (#35) Wanted to use some of the setup for bootstrapping another project, so I had a close look over these couple of items. Reviewed-on: https://git.data.coop/data.coop/membersystem/pulls/35 Reviewed-by: valberg Co-authored-by: Benjamin Bach Co-committed-by: Benjamin Bach --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fe342b7..fc8aa79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,15 @@ FROM python:3.12-slim-bullseye +# PYTHONFAULTHANDLER: Propagate tracebacks from all threads. +# PYTHONUNBUFFERED: Write terminal output straight to docker (to not confuse Docker Compose). +# PYTHONDONTWRITEBYTECODE: Dont write *pyc files at all, making it possible for a 100% read-only container. +# PIP_NO_CACHE_DIR: Disable PIP cache, we don't need pip's cache after building the image. +# PIP_DISABLE_PIP_VERSION_CHECK: Build the image with the available pip, do not check for updates (faster!) +# PIP_DEFAULT_TIMEOUT: Allow for longer timeouts. ENV PYTHONFAULTHANDLER=1 \ PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ - PYTHONHASHSEED=random \ - PIP_NO_CACHE_DIR=off \ + PIP_NO_CACHE_DIR=1 \ PIP_DISABLE_PIP_VERSION_CHECK=on \ PIP_DEFAULT_TIMEOUT=100 ARG BUILD From f6d8f820656a94091d17ca7408219d338f70355d Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Wed, 31 Jul 2024 21:17:00 +0000 Subject: [PATCH 2/4] Requirements pinning + some cleanup (#36) Reviewed-on: https://git.data.coop/data.coop/membersystem/pulls/36 Reviewed-by: valberg Co-authored-by: Benjamin Bach Co-committed-by: Benjamin Bach --- .pre-commit-config.yaml | 2 +- pyproject.toml | 20 +++++++------- requirements.txt | 46 ++++++++++++++----------------- requirements/requirements-dev.txt | 18 ++++++------ 4 files changed, 40 insertions(+), 46 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49e1ce6..7388e5d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.12 + python: python3 exclude: ^.*\b(migrations)\b.*$ repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/pyproject.toml b/pyproject.toml index 643b1cd..df44311 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,17 +12,17 @@ authors = [ { name = "Víðir Valberg Guðmundsson", email = "valberg@orn.li" }, ] dependencies = [ - "Django==5.0.7", - "django-money==3.5.2", - "django-allauth==0.63.6", - "psycopg[binary]==3.2.1", - "environs[django]==11.0.0", - "uvicorn==0.30.1", - "whitenoise==6.7.0", - "django-zen-queries==2.1.0", + "Django~=5.0", + "django-money~=3.5", + "django-allauth~=0.63", + "psycopg[binary]~=3.2", + "environs[django]>=11,<12", + "uvicorn~=0.30", + "whitenoise~=6.7", + "django-zen-queries~=2.1", "django-registries==0.0.3", "django-view-decorator==0.0.4", - "django-oauth-toolkit==2.4.0", + "django-oauth-toolkit~=2.4", ] version = "0.0.1" @@ -66,7 +66,7 @@ matrix.python.dependencies = [ cov = "pytest --cov-report=term-missing --cov-config=pyproject.toml --cov=src --cov=tests --cov=append {args}" no-cov = "cov --no-cov {args}" typecheck = "mypy --config-file=pyproject.toml ." -requirements = "pip-compile --output-file requirements/base.txt pyproject.toml" +requirements = "pip-compile pyproject.toml" server = "./src/manage.py runserver 0.0.0.0:8000" migrate = "./src/manage.py migrate" makemigrations = "./src/manage.py makemigrations" diff --git a/requirements.txt b/requirements.txt index 5642b3a..b36b955 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,9 @@ # -# This file is autogenerated by hatch-pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: # -# - django-allauth==0.63.6 -# - django-money==3.5.2 -# - django-oauth-toolkit==2.4.0 -# - django-registries==0.0.3 -# - django-view-decorator==0.0.4 -# - django-zen-queries==2.1.0 -# - django==5.0.7 -# - environs[django]==11.0.0 -# - psycopg[binary]==3.2.1 -# - uvicorn==0.30.1 -# - whitenoise==6.7.0 +# pip-compile pyproject.toml # - asgiref==3.8.1 # via django babel==2.15.0 @@ -34,7 +24,6 @@ dj-email-url==1.0.6 # via environs django==5.0.7 # via - # hatch.envs.default # dj-database-url # django-allauth # django-money @@ -42,22 +31,25 @@ django==5.0.7 # django-registries # django-view-decorator # django-zen-queries + # membersystem (pyproject.toml) django-allauth==0.63.6 - # via hatch.envs.default + # via membersystem (pyproject.toml) django-cache-url==3.4.5 # via environs django-money==3.5.2 - # via hatch.envs.default + # via membersystem (pyproject.toml) django-oauth-toolkit==2.4.0 - # via hatch.envs.default + # via membersystem (pyproject.toml) django-registries==0.0.3 - # via hatch.envs.default + # via membersystem (pyproject.toml) django-view-decorator==0.0.4 - # via hatch.envs.default + # via membersystem (pyproject.toml) django-zen-queries==2.1.0 - # via hatch.envs.default -environs==11.0.0 - # via hatch.envs.default + # via membersystem (pyproject.toml) +environs[django]==11.0.0 + # via + # environs + # membersystem (pyproject.toml) h11==0.14.0 # via uvicorn idna==3.7 @@ -70,8 +62,10 @@ oauthlib==3.2.2 # via django-oauth-toolkit packaging==24.1 # via marshmallow -psycopg==3.2.1 - # via hatch.envs.default +psycopg[binary]==3.2.1 + # via + # membersystem (pyproject.toml) + # psycopg psycopg-binary==3.2.1 # via psycopg py-moneyed==3.0 @@ -95,9 +89,9 @@ typing-extensions==4.12.2 urllib3==2.2.2 # via requests uvicorn==0.30.1 - # via hatch.envs.default + # via membersystem (pyproject.toml) whitenoise==6.7.0 - # via hatch.envs.default + # via membersystem (pyproject.toml) # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index f044e8c..88d018a 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -11,17 +11,17 @@ # - django-debug-toolbar==4.2.0 # - django-browser-reload==1.7.0 # - model-bakery==1.17.0 -# - django-allauth==0.63.6 -# - django-money==3.5.2 -# - django-oauth-toolkit==2.4.0 +# - django-allauth~=0.63 +# - django-money~=3.5 +# - django-oauth-toolkit~=2.4 # - django-registries==0.0.3 # - django-view-decorator==0.0.4 -# - django-zen-queries==2.1.0 -# - django==5.0.7 -# - environs[django]==11.0.0 -# - psycopg[binary]==3.2.1 -# - uvicorn==0.30.1 -# - whitenoise==6.7.0 +# - django-zen-queries~=2.1 +# - django~=5.0 +# - environs[django]<12,>=11 +# - psycopg[binary]~=3.2 +# - uvicorn~=0.30 +# - whitenoise~=6.7 # asgiref==3.8.1 From 7a3a629d6f7a27073cd5ec83492dd870aa581157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=AD=C3=B0ir=20Valberg=20Gu=C3=B0mundsson?= Date: Wed, 31 Jul 2024 23:26:40 +0200 Subject: [PATCH 3/4] Update requirements compilation to use hatch-pip-compile. --- README.md | 10 +++++++ pyproject.toml | 4 +-- requirements.txt | 50 +++++++++++++++++-------------- requirements/requirements-dev.txt | 17 ++++------- 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 2c8d5d3..2524f93 100644 --- a/README.md +++ b/README.md @@ -72,3 +72,13 @@ hatch run dev:migrate ```bash hatch run dev:server ``` + +#### Updating requirements + +If you want to update the requirements, you can run the following command: + +```bash +hatch run requirements +``` + +This uses [hatch-pip-compile](https://juftin.com/hatch-pip-compile/) to update the requirements. diff --git a/pyproject.toml b/pyproject.toml index df44311..fc88792 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,11 +62,11 @@ matrix.python.dependencies = [ { value = "typing_extensions==4.5.0", if = ["3.10"]}, ] -[tool.hatch.envs.dev.scripts] +[tool.hatch.envs.default.scripts] cov = "pytest --cov-report=term-missing --cov-config=pyproject.toml --cov=src --cov=tests --cov=append {args}" no-cov = "cov --no-cov {args}" typecheck = "mypy --config-file=pyproject.toml ." -requirements = "pip-compile pyproject.toml" +requirements = "hatch env run --env default -- python --version; hatch env run --env dev -- python --version" server = "./src/manage.py runserver 0.0.0.0:8000" migrate = "./src/manage.py migrate" makemigrations = "./src/manage.py makemigrations" diff --git a/requirements.txt b/requirements.txt index b36b955..5052e2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,19 @@ # -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: +# This file is autogenerated by hatch-pip-compile with Python 3.12 # -# pip-compile pyproject.toml +# - django-allauth~=0.63 +# - django-money~=3.5 +# - django-oauth-toolkit~=2.4 +# - django-registries==0.0.3 +# - django-view-decorator==0.0.4 +# - django-zen-queries~=2.1 +# - django~=5.0 +# - environs[django]<12,>=11 +# - psycopg[binary]~=3.2 +# - uvicorn~=0.30 +# - whitenoise~=6.7 # + asgiref==3.8.1 # via django babel==2.15.0 @@ -16,7 +26,7 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via uvicorn -cryptography==42.0.8 +cryptography==43.0.0 # via jwcrypto dj-database-url==2.2.0 # via environs @@ -24,6 +34,7 @@ dj-email-url==1.0.6 # via environs django==5.0.7 # via + # hatch.envs.default # dj-database-url # django-allauth # django-money @@ -31,25 +42,22 @@ django==5.0.7 # django-registries # django-view-decorator # django-zen-queries - # membersystem (pyproject.toml) django-allauth==0.63.6 - # via membersystem (pyproject.toml) + # via hatch.envs.default django-cache-url==3.4.5 # via environs django-money==3.5.2 - # via membersystem (pyproject.toml) + # via hatch.envs.default django-oauth-toolkit==2.4.0 - # via membersystem (pyproject.toml) + # via hatch.envs.default django-registries==0.0.3 - # via membersystem (pyproject.toml) + # via hatch.envs.default django-view-decorator==0.0.4 - # via membersystem (pyproject.toml) + # via hatch.envs.default django-zen-queries==2.1.0 - # via membersystem (pyproject.toml) -environs[django]==11.0.0 - # via - # environs - # membersystem (pyproject.toml) + # via hatch.envs.default +environs==11.0.0 + # via hatch.envs.default h11==0.14.0 # via uvicorn idna==3.7 @@ -62,10 +70,8 @@ oauthlib==3.2.2 # via django-oauth-toolkit packaging==24.1 # via marshmallow -psycopg[binary]==3.2.1 - # via - # membersystem (pyproject.toml) - # psycopg +psycopg==3.2.1 + # via hatch.envs.default psycopg-binary==3.2.1 # via psycopg py-moneyed==3.0 @@ -88,10 +94,10 @@ typing-extensions==4.12.2 # py-moneyed urllib3==2.2.2 # via requests -uvicorn==0.30.1 - # via membersystem (pyproject.toml) +uvicorn==0.30.3 + # via hatch.envs.default whitenoise==6.7.0 - # via membersystem (pyproject.toml) + # via hatch.envs.default # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index 88d018a..39faa96 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -45,9 +45,8 @@ click==8.1.7 coverage==7.3.0 # via # hatch.envs.dev - # coverage # pytest-cov -cryptography==42.0.8 +cryptography==43.0.0 # via jwcrypto dj-database-url==2.2.0 # via environs @@ -84,16 +83,14 @@ django-registries==0.0.3 # via hatch.envs.dev django-stubs==1.16.0 # via hatch.envs.dev -django-stubs-ext==5.0.2 +django-stubs-ext==5.0.4 # via django-stubs django-view-decorator==0.0.4 # via hatch.envs.dev django-zen-queries==2.1.0 # via hatch.envs.dev environs==11.0.0 - # via - # hatch.envs.dev - # environs + # via hatch.envs.dev h11==0.14.0 # via uvicorn idna==3.7 @@ -124,9 +121,7 @@ pip-tools==7.3.0 pluggy==1.5.0 # via pytest psycopg==3.2.1 - # via - # hatch.envs.dev - # psycopg + # via hatch.envs.dev psycopg-binary==3.2.1 # via psycopg py-moneyed==3.0 @@ -158,7 +153,7 @@ tomli==2.0.1 # via django-stubs types-pytz==2024.1.0.20240417 # via django-stubs -types-pyyaml==6.0.12.20240311 +types-pyyaml==6.0.12.20240724 # via django-stubs typing-extensions==4.12.2 # via @@ -171,7 +166,7 @@ typing-extensions==4.12.2 # py-moneyed urllib3==2.2.2 # via requests -uvicorn==0.30.1 +uvicorn==0.30.3 # via hatch.envs.dev wheel==0.43.0 # via pip-tools From 0cf579c5f6e70b837c487e254532987d7186a56f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=AD=C3=B0ir=20Valberg=20Gu=C3=B0mundsson?= Date: Wed, 31 Jul 2024 23:48:39 +0200 Subject: [PATCH 4/4] Update dockerfile to using bookworm, and to avoid invalidating cache unless updating requirements. --- Dockerfile | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index fc8aa79..9b7c01b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim-bullseye +FROM python:3.12-slim-bookworm # PYTHONFAULTHANDLER: Propagate tracebacks from all threads. # PYTHONUNBUFFERED: Write terminal output straight to docker (to not confuse Docker Compose). @@ -13,14 +13,17 @@ ENV PYTHONFAULTHANDLER=1 \ PIP_DISABLE_PIP_VERSION_CHECK=on \ PIP_DEFAULT_TIMEOUT=100 ARG BUILD -ENV BUILD ${BUILD} +ENV BUILD=${BUILD} ARG REQUIREMENTS_FILE=requirements.txt WORKDIR /app RUN groupadd -g 1000 www && useradd -u 1000 -ms /bin/bash -g www www -COPY --chown=www:www . . -RUN mkdir /app/src/static && \ + +# Only copy the requirements file first to leverage Docker cache +COPY $REQUIREMENTS_FILE . + +RUN mkdir -p /app/src/static && \ chown www:www /app/src/static && \ apt-get update && \ apt-get install -y \ @@ -35,8 +38,12 @@ RUN mkdir /app/src/static && \ libffi-dev \ shared-mime-info \ gettext && \ - pip install --no-cache-dir -r $REQUIREMENTS_FILE && \ - django-admin compilemessages + pip install --no-cache-dir -r $REQUIREMENTS_FILE + +# Copy the rest of the application +COPY . . + +RUN django-admin compilemessages ENTRYPOINT ["./entrypoint.sh"]