From 388941cd8a244d1046a1d1aaea39d0b6c8b9bb3c Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Thu, 18 Apr 2019 09:44:12 -0400 Subject: [PATCH 1/7] Update Ubuntu and other dependencies We update our base image to Ubuntu 18.04, and update all of our dependencies to the latest released versions. This includes: - gcc-multilib-arm-linux-gnueabihf (latest distro version) - mdbook 0.2.1 (0.2.3 is out, but there's no binary build) - OpenSSL 1.1.1b - libpq 11.2 --- Dockerfile | 26 +++++++++++++------------- cargo-config.toml | 2 +- hooks/build | 21 +++++++++++++++------ hooks/test | 9 +++++++++ 4 files changed, 38 insertions(+), 20 deletions(-) create mode 100755 hooks/test diff --git a/Dockerfile b/Dockerfile index cc5ef5a..59912d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ -# Use Debian 16.04 as the base for our Rust musl toolchain, because of -# https://github.com/rust-lang/rust/issues/34978 (as of Rust 1.11). -FROM ubuntu:16.04 +# Use Ubuntu 18.04 LTS as our base image. +FROM ubuntu:18.04 # The Rust toolchain to use when building our image. Set by `hooks/build`. ARG TOOLCHAIN=stable @@ -29,11 +28,11 @@ RUN apt-get update && \ pkgconf \ sudo \ xutils-dev \ - gcc-4.7-multilib-arm-linux-gnueabihf \ + gcc-multilib-arm-linux-gnueabihf \ && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ useradd rust --user-group --create-home --shell /bin/bash --groups sudo && \ - MDBOOK_VERSION=0.1.5 && \ + MDBOOK_VERSION=0.2.1 && \ curl -LO https://github.com/rust-lang-nursery/mdBook/releases/download/v$MDBOOK_VERSION/mdbook-v$MDBOOK_VERSION-x86_64-unknown-linux-musl.tar.gz && \ tar xf mdbook-v$MDBOOK_VERSION-x86_64-unknown-linux-musl.tar.gz && \ mv mdbook /usr/local/bin/ && \ @@ -74,30 +73,31 @@ RUN git config --global credential.https://github.com.helper ghtoken # needed by the popular Rust `hyper` crate. RUN echo "Building OpenSSL" && \ cd /tmp && \ - OPENSSL_VERSION=1.0.2o && \ + OPENSSL_VERSION=1.1.1b && \ curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ tar xvzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ - env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl linux-x86_64 && \ + env CC=musl-gcc ./Configure no-shared no-zlib no-async no-engine -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make depend && \ make && sudo make install && \ - \ - echo "Building zlib" && \ + rm -r /tmp/* + +RUN echo "Building zlib" && \ cd /tmp && \ ZLIB_VERSION=1.2.11 && \ curl -LO "http://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \ tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \ CC=musl-gcc ./configure --static --prefix=/usr/local/musl && \ make && sudo make install && \ - \ - echo "Building libpq" && \ + rm -r /tmp/* + +RUN echo "Building libpq" && \ cd /tmp && \ - POSTGRESQL_VERSION=9.6.8 && \ + POSTGRESQL_VERSION=11.2 && \ curl -LO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \ tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \ CC=musl-gcc CPPFLAGS=-I/usr/local/musl/include LDFLAGS=-L/usr/local/musl/lib ./configure --with-openssl --without-readline --prefix=/usr/local/musl && \ cd src/interfaces/libpq && make all-static-lib && sudo make install-lib-static && \ cd ../../bin/pg_config && make && sudo make install && \ - \ rm -r /tmp/* ENV OPENSSL_DIR=/usr/local/musl/ \ diff --git a/cargo-config.toml b/cargo-config.toml index 6208b7b..848fe3a 100644 --- a/cargo-config.toml +++ b/cargo-config.toml @@ -3,4 +3,4 @@ target = "x86_64-unknown-linux-musl" [target.armv7-unknown-linux-musleabihf] -linker = "arm-linux-gnueabihf-gcc-4.7" +linker = "arm-linux-gnueabihf-gcc" diff --git a/hooks/build b/hooks/build index a55cb8c..25810b6 100755 --- a/hooks/build +++ b/hooks/build @@ -3,12 +3,21 @@ # Abort if anything goes wrong. set -euo pipefail -# Always map the Docker tag `latest` to stable Rust. -if [ "$DOCKER_TAG" == "latest" ]; then - TOOLCHAIN=stable -else - TOOLCHAIN="$DOCKER_TAG" -fi +# Decide what Rust toolchain to use. +case "$DOCKER_TAG" in + # Always map the Docker tags `latest` and `experimental` to stable Rust. + latest|experimental) + TOOLCHAIN=stable + ;; + # Strip `experimental-` from other `experimental-*` tags. + experimental-*) + TOOLCHAIN="${DOCKER_TAG/experimental-/}" + ;; + # Pass all our tags + *) + TOOLCHAIN="$DOCKER_TAG" + ;; +esac # Run the build. docker build --build-arg TOOLCHAIN="$TOOLCHAIN" -t "$IMAGE_NAME" . diff --git a/hooks/test b/hooks/test new file mode 100755 index 0000000..ca536cc --- /dev/null +++ b/hooks/test @@ -0,0 +1,9 @@ +#!/bin/bash + +# Abort if anything goes wrong. +set -euo pipefail + +# Make sure we can build some of our more important test images. +for EXAMPLE in using-diesel; do + docker build -t rust-musl-builder-"$EXAMPLE" examples/"$EXAMPLE" +done From e50cbda16910bf90c1d00b1d4b0306992f9ab1cf Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sat, 27 Apr 2019 08:51:12 -0400 Subject: [PATCH 2/7] Allow building with OpenSSL 1.0 or 1.1 It turns out that some popular libaries still have problems with OpenSSL 1.1, particularly `postgres` 0.15. So we'll build two sets of images now. We also explain the new image tagging scheme, and fix a bug wher the test script didn't test the newly built image. --- Dockerfile | 5 ++++- README.md | 23 ++++++++++++++++++++ examples/using-diesel/Dockerfile | 6 +++++- hooks/build | 36 ++++++++++++++++++++++++++------ hooks/test | 5 ++++- 5 files changed, 66 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 59912d3..a3accb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,10 @@ FROM ubuntu:18.04 # The Rust toolchain to use when building our image. Set by `hooks/build`. ARG TOOLCHAIN=stable +# The OpenSSL version to use. We parameterize this because many Rust +# projects will fail to build with 1.1. +ARG OPENSSL_VERSION=1.0.2r + # Make sure we have basic dev tools for building C libraries. Our goal # here is to support the musl-libc builds and Cargo builds needed for a # large selection of the most popular crates. @@ -73,7 +77,6 @@ RUN git config --global credential.https://github.com.helper ghtoken # needed by the popular Rust `hyper` crate. RUN echo "Building OpenSSL" && \ cd /tmp && \ - OPENSSL_VERSION=1.1.1b && \ curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ tar xvzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ env CC=musl-gcc ./Configure no-shared no-zlib no-async no-engine -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ diff --git a/README.md b/README.md index d95d0df..5135e9a 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,29 @@ Binaries will be written to `target/$TARGET_ARCHITECTURE/release`. By default it With a bit of luck, you should be able to just copy your application binary from `target/x86_64-unknown-linux-musl/release`, and install it directly on any reasonably modern x86_64 Linux machine. In particular, you should be able make static release binaries using TravisCI and GitHub, or you can copy your Rust application into an [Alpine Linux container][]. See below for details! +## Available tags + +In general, we provide the following tagged Docker images: + +- `latest`, `stable`: Current stable Rust, with OpenSSL 1.0 (for now). We + try to update this fairly rapidly after every new stable release, and + after most point releases. +- `beta`: This usually gets updated every six weeks alongside the stable + release. It will usually not be updated for beta bugfix releases. +- `nightly-YYYY-MM-DD`: Specific nightly releases. These should almost + always support `clippy`, `rls` and `rustfmt`, as verified using + [rustup components history][comp]. If you need a specific date for + compatibility with `tokio` or another popular library using unstable + Rust, please file an issue. +- `stable-openssl11`: Current stable Rust, with OpenSSL 1.1. +- `nightly-YYYY-MM-DD-openssl11`: Specific nightly releases with OpenSSL + 1.1. + +At a minimum, each of these images should be able to +compile [examples/using-diesel](./examples/using-diesel). + +[comp]: https://rust-lang.github.io/rustup-components-history/index.html + ## Caching builds You may be able to speed up build performance by adding the following `-v` commands to the `rust-musl-builder` alias: diff --git a/examples/using-diesel/Dockerfile b/examples/using-diesel/Dockerfile index dc3406a..634a521 100644 --- a/examples/using-diesel/Dockerfile +++ b/examples/using-diesel/Dockerfile @@ -3,8 +3,12 @@ # An example Dockerfile showing how to build a Rust executable using this # image, and deploy it with a tiny Alpine Linux container. +# You can override this `--build-arg BASE_IMAGE=...` to use different +# version of Rust or OpenSSL. +ARG BASE_IMAGE=ekidd/rust-musl-builder:latest + # Our first FROM statement declares the build environment. -FROM ekidd/rust-musl-builder AS builder +FROM ${BASE_IMAGE} AS builder # Add our source code. ADD . ./ diff --git a/hooks/build b/hooks/build index 25810b6..8aa6213 100755 --- a/hooks/build +++ b/hooks/build @@ -1,23 +1,47 @@ #!/bin/bash +# +# This script is responsible for calling `docker build` on behalf of GitHub. But +# what it _really_ does is translate `DOCKER_TAG` into the builds args `TARGET` +# and `OPENSSL_VERSION`. # Abort if anything goes wrong. set -euo pipefail -# Decide what Rust toolchain to use. +# Default to using OpenSSL 1.0 for a while longer, because 1.1 is +# incompatible with the crates postgres 0.15 and openssl 0.9, which +# are still widely used. +OPENSSL_VERSION=1.0.2r + +# Pick an appropriate Docker tag case "$DOCKER_TAG" in + *-openssl11) + DOCKER_TAG_WITHOUT_OPENSSL="${DOCKER_TAG/-openssl11/}" + OPENSSL_VERSION=1.1.1b + ;; + *) + DOCKER_TAG_WITHOUT_OPENSSL="$DOCKER_TAG" + ;; +esac + +# Decide what Rust toolchain to use. +case "$DOCKER_TAG_WITHOUT_OPENSSL" in # Always map the Docker tags `latest` and `experimental` to stable Rust. - latest|experimental) + latest) TOOLCHAIN=stable ;; # Strip `experimental-` from other `experimental-*` tags. experimental-*) - TOOLCHAIN="${DOCKER_TAG/experimental-/}" + TOOLCHAIN="${DOCKER_TAG_WITHOUT_OPENSSL/experimental-/}" ;; - # Pass all our tags + # Pass all other tags through. *) - TOOLCHAIN="$DOCKER_TAG" + TOOLCHAIN="$DOCKER_TAG_WITHOUT_OPENSSL" ;; esac # Run the build. -docker build --build-arg TOOLCHAIN="$TOOLCHAIN" -t "$IMAGE_NAME" . +docker build \ + --build-arg TOOLCHAIN="$TOOLCHAIN" \ + --build-arg OPENSSL_VERSION="$OPENSSL_VERSION" \ + -t "$IMAGE_NAME" \ + . diff --git a/hooks/test b/hooks/test index ca536cc..72d6362 100755 --- a/hooks/test +++ b/hooks/test @@ -5,5 +5,8 @@ set -euo pipefail # Make sure we can build some of our more important test images. for EXAMPLE in using-diesel; do - docker build -t rust-musl-builder-"$EXAMPLE" examples/"$EXAMPLE" + docker build \ + --build-arg BASE_IMAGE="$IMAGE_NAME" \ + -t rust-musl-builder-"$EXAMPLE" \ + examples/"$EXAMPLE" done From 58098af689a4ab3ecc1c1406cbddb0f86398dd29 Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sat, 27 Apr 2019 17:53:16 -0400 Subject: [PATCH 3/7] OpenSSL: Re-enable `async` and `engine` We can't build `libgit2` while passing `no-engine` to OpenSSL. Let's also try removing `no-async`, just in case somebody needs that. `no-engine` seems to break OpenSSL 1.0 but not 1.1. I'm not feeling motivated to find out why on a Saturday; let the CI system see if this works. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a3accb9..0534013 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,7 +79,7 @@ RUN echo "Building OpenSSL" && \ cd /tmp && \ curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ tar xvzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ - env CC=musl-gcc ./Configure no-shared no-zlib no-async no-engine -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ + env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make depend && \ make && sudo make install && \ rm -r /tmp/* From e58507e1aa59e3f9cfbe6e3986f943f8f7dad83b Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sat, 27 Apr 2019 21:32:09 -0400 Subject: [PATCH 4/7] Attempt to kludge a fix for OpenSSL 1.1 "engine" This is needed to build `cargo audit` and other programs that use OpenSSL 1.1's "engine" component. --- Dockerfile | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0534013..3d13875 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ RUN apt-get update && \ libpq-dev \ libsqlite-dev \ libssl-dev \ + linux-libc-dev \ pkgconf \ sudo \ xutils-dev \ @@ -73,15 +74,27 @@ ADD cargo-config.toml /home/rust/.cargo/config ADD git-credential-ghtoken /usr/local/bin RUN git config --global credential.https://github.com.helper ghtoken -# Build a static library version of OpenSSL using musl-libc. This is -# needed by the popular Rust `hyper` crate. +# Build a static library version of OpenSSL using musl-libc. This is needed by +# the popular Rust `hyper` crate. +# +# We point /usr/local/musl/include/linux at some Linux kernel headers (not +# necessarily the right ones) in an effort to compile OpenSSL 1.1's "engine" +# component. It's possible that this will cause bizarre and terrible things to +# happen. There may be "sanitized" header RUN echo "Building OpenSSL" && \ + ls /usr/include/linux && \ + sudo mkdir -p /usr/local/musl/include && \ + sudo ln -s /usr/include/linux /usr/local/musl/include/linux && \ + sudo ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/musl/include/asm && \ + sudo ln -s /usr/include/asm-generic /usr/local/musl/include/asm-generic && \ cd /tmp && \ curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \ tar xvzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \ env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make depend && \ - make && sudo make install && \ + env C_INCLUDE_PATH=/usr/local/musl/include/ make && \ + sudo make install && \ + sudo rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ rm -r /tmp/* RUN echo "Building zlib" && \ From 6a0705ca9d32abab341dee85d7452268ae74ecc5 Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sun, 28 Apr 2019 13:37:11 -0400 Subject: [PATCH 5/7] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 2 -- .github/ISSUE_TEMPLATE/build_request.md | 12 ++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 4 ++-- 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/build_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 28e17cd..983f3c1 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -19,8 +19,6 @@ Please include the steps you took, including the commands you ran in your termin **What did you hope to happen?** -(A clear and concise description of what you expected to happen.) - **Does `./test-image` work?** If you check out the repository on a Mac or Linux system and run `./build-image`, does it succeed or fail? This will build several known-good examples from the `rust-musl-builder/examples` directory. diff --git a/.github/ISSUE_TEMPLATE/build_request.md b/.github/ISSUE_TEMPLATE/build_request.md new file mode 100644 index 0000000..3053fc8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/build_request.md @@ -0,0 +1,12 @@ +--- +name: Build request +about: Does the image need to be rebuilt? +title: '' +labels: bug +assignees: emk + +--- + +**Rust release:** (stable, X.Y.Z, beta, nightly-YYYY-MM-DD) + +**OpenSSL:** 1.0/1.1 (pick one) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index a9a280f..9644b1a 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -4,10 +4,10 @@ about: I like hearing suggestions (but I don't always have time to implement the unfortunatey). title: '' labels: enhancement -assignees: '' +assignees: emk --- **How could this project be improved?** -**If you're interested in implementing this feature yourself, would you like input or mentoring?** +**If you're interested in implementing this feature yourself, how could I help you?** From 17db9d6272cdd1497911138097d4c245566fefbb Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sun, 28 Apr 2019 13:37:21 -0400 Subject: [PATCH 6/7] Update README for new images --- README.md | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 5135e9a..caccd72 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ [![Docker Image](https://img.shields.io/docker/pulls/ekidd/rust-musl-builder.svg?maxAge=2592000)](https://hub.docker.com/r/ekidd/rust-musl-builder/) +**NOTE:** The underlying build image is now running Ubuntu 18.04 and newer +versions of several libraries. Please report any problems! + Do you want to compile a completely static Rust binary with no external dependencies? If so, try: ```sh @@ -11,13 +14,7 @@ rust-musl-builder cargo build --release This command assumes that `$(pwd)` is readable and writable by uid 1000, gid 1000. At the moment, it doesn't attempt to cache libraries between builds, so this is best reserved for making final release builds. -To target ARM hard float (Raspberry Pi): - -```sh -rust-musl-builder cargo build --target=armv7-unknown-linux-musleabihf --release -``` - -Binaries will be written to `target/$TARGET_ARCHITECTURE/release`. By default it targets `x86_64-unknown-linux-musl` unless specified with `--target`. +For a more realistic example, see the `Dockerfile` for [examples/using-diesel](./examples/using-diesel). ## Deploying your Rust application @@ -30,6 +27,7 @@ In general, we provide the following tagged Docker images: - `latest`, `stable`: Current stable Rust, with OpenSSL 1.0 (for now). We try to update this fairly rapidly after every new stable release, and after most point releases. +- `X.Y.Z`: Specific versions of stable Rust. - `beta`: This usually gets updated every six weeks alongside the stable release. It will usually not be updated for beta bugfix releases. - `nightly-YYYY-MM-DD`: Specific nightly releases. These should almost @@ -38,7 +36,8 @@ In general, we provide the following tagged Docker images: compatibility with `tokio` or another popular library using unstable Rust, please file an issue. - `stable-openssl11`: Current stable Rust, with OpenSSL 1.1. -- `nightly-YYYY-MM-DD-openssl11`: Specific nightly releases with OpenSSL +- `X.Y.Z-openssl11`: Specific versions of stable Rust, with OpenSSL 1.1. +- `nightly-YYYY-MM-DD-openssl11`: Specific nightly releases, with OpenSSL 1.1. At a minimum, each of these images should be able to @@ -186,6 +185,18 @@ If you're using Docker crates which require specific C libraries to be installed If you need an especially common library, please feel free to submit a pull request adding it to the main `Dockerfile`! We'd like to support popular Rust crates out of the box. +## ARM support (experimental) + +To target ARM hard float (Raspberry Pi): + +```sh +rust-musl-builder cargo build --target=armv7-unknown-linux-musleabihf --release +``` + +Binaries will be written to `target/$TARGET_ARCHITECTURE/release`. By default it targets `x86_64-unknown-linux-musl` unless specified with `--target`. + +This is missing many of the libraries used by the `x86_64` build, and it should probably be split out of the base image and given its own tags. + ## Development notes After modifying the image, run `./test-image` to make sure that everything works.xs From 318c95d3654ce4bcd0065fa1cee95da574b06435 Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sun, 28 Apr 2019 13:55:14 -0400 Subject: [PATCH 7/7] Remove obsolete note --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index caccd72..3d1b28f 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ rust-musl-builder sudo chown -R rust:rust \ - The standard `musl-libc` libraries. - OpenSSL, which is needed by many Rust applications. -- `libpq`, which is needed for applications that use `diesel` with PostgreSQL. Note that this may be broken under Rust 1.21.0 and later (see https://github.com/emk/rust-musl-builder/issues/27). +- `libpq`, which is needed for applications that use `diesel` with PostgreSQL. - `libz`, which is needed by `libpq`. - SQLite3. See [examples/using-diesel](./examples/using-diesel/).