From 3ae8179705d6d8bf707692c5e3cd8970a9753872 Mon Sep 17 00:00:00 2001 From: Eric Kidd Date: Sun, 6 Sep 2020 11:48:56 -0400 Subject: [PATCH] Experimental: Improve support for `USER root` This PR attempts to lay the groundwork to address https://github.com/emk/rust-musl-builder/issues/96. In particular, we install the Rust toolchain globally by abusing `rustup`. We still preserve the legacy `rust` user but we now encourage derived Dockerfiles to consider `USER root`. This PR also removes ARM support. We want to explore the idea of supporting ARM more reliably using a separate image, but perhaps it would be better to refer those users to one of the Rust cross-compilation toolchains. See https://github.com/emk/rust-musl-builder/issues/63 for discussion. --- CHANGELOG.md | 15 +++++++ Dockerfile | 123 +++++++++++++++++++++++++++++---------------------- test-image | 26 ----------- 3 files changed, 84 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dfb452..edd22ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). For maximum stablity, use images with tags like `ekidd/rust-musl-builder:1.46.0` or `ekidd/rust-musl-builder:nightly-2020-08-26`. These may occasionally be rebuilt, but only while they're "current", or possibly if they're recent and serious security are discovered in a library. +## [UNRELEASED] + +This branch contains experimental changes intended to make it easier to support GitHub Actions. + +### Changed + +- You'll need to use `USER root` and `env RUSTUP_HOME=/opt/rust/rustup CARGO_HOME=/opt/rust/cargo rustup $ARGS` to install any new components. +- `rustup`, `cargo`, and associated tools are all installed in `/opt/rust`, so that they should be available to the users `rust`, `root`, and any other users that get added. +- Some other minor supporting tools like `git-credential-ghtoken` should now be available as `root`, as well. + +### Removed + +- ARM support has been removed, because it needs to be split into a separate base image. This would also allow us to build OpenSSL, etc., for ARM targets. +- The `rust-docs` component is no longer installed by default. + ## 2020-09-04 ### Added diff --git a/Dockerfile b/Dockerfile index 1089312..eec70f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,13 +26,12 @@ ARG CARGO_DENY_VERSION=0.7.3 ARG ZLIB_VERSION=1.2.11 ARG POSTGRESQL_VERSION=11.9 -# Make sure we have basic dev tools for building C libraries. Our goal -# here is to support the musl-libc builds and Cargo builds needed for a -# large selection of the most popular crates. +# Make sure we have basic dev tools for building C libraries. Our goal here is +# to support the musl-libc builds and Cargo builds needed for a large selection +# of the most popular crates. # -# We also set up a `rust` user by default, in whose account we'll install -# the Rust toolchain. This user has sudo privileges if you need to install -# any more software. +# We also set up a `rust` user by default. This user has sudo privileges if you +# need to install any more software. # # `mdbook` is the standard Rust tool for making searchable HTML manuals. RUN apt-get update && \ @@ -52,7 +51,6 @@ RUN apt-get update && \ pkgconf \ sudo \ xutils-dev \ - gcc-multilib-arm-linux-gnueabihf \ && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ useradd rust --user-group --create-home --shell /bin/bash --groups sudo && \ @@ -70,35 +68,7 @@ RUN apt-get update && \ rm -rf cargo-deny-$CARGO_DENY_VERSION-x86_64-unknown-linux-musl cargo-deny-$CARGO_DENY_VERSION-x86_64-unknown-linux-musl.tar.gz # Static linking for C++ code -RUN sudo ln -s "/usr/bin/g++" "/usr/bin/musl-g++" - -# Allow sudo without a password. -ADD sudoers /etc/sudoers.d/nopasswd - -# Run all further code as user `rust`, and create our working directories -# as the appropriate user. -USER rust -RUN mkdir -p /home/rust/libs /home/rust/src - -# Set up our path with all our binary directories, including those for the -# musl-gcc toolchain and for our Rust toolchain. -ENV PATH=/home/rust/.cargo/bin:/usr/local/musl/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - -# Install our Rust toolchain and the `musl` target. We patch the -# command-line we pass to the installer so that it won't attempt to -# interact with the user or fool around with TTYs. We also set the default -# `--target` to musl so that our users don't need to keep overriding it -# manually. -RUN curl https://sh.rustup.rs -sSf | \ - sh -s -- -y --default-toolchain $TOOLCHAIN && \ - rustup target add x86_64-unknown-linux-musl && \ - rustup target add armv7-unknown-linux-musleabihf -ADD cargo-config.toml /home/rust/.cargo/config - -# Set up a `git credentials` helper for using GH_USER and GH_TOKEN to access -# private repositories if desired. -ADD git-credential-ghtoken /usr/local/bin/ghtoken -RUN git config --global credential.https://github.com.helper ghtoken +RUN ln -s "/usr/bin/g++" "/usr/bin/musl-g++" # Build a static library version of OpenSSL using musl-libc. This is needed by # the popular Rust `hyper` crate. @@ -109,10 +79,10 @@ RUN git config --global credential.https://github.com.helper ghtoken # happen. There may be "sanitized" header RUN echo "Building OpenSSL" && \ ls /usr/include/linux && \ - sudo mkdir -p /usr/local/musl/include && \ - sudo ln -s /usr/include/linux /usr/local/musl/include/linux && \ - sudo ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/musl/include/asm && \ - sudo ln -s /usr/include/asm-generic /usr/local/musl/include/asm-generic && \ + mkdir -p /usr/local/musl/include && \ + ln -s /usr/include/linux /usr/local/musl/include/linux && \ + ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/musl/include/asm && \ + ln -s /usr/include/asm-generic /usr/local/musl/include/asm-generic && \ cd /tmp && \ short_version="$(echo "$OPENSSL_VERSION" | sed s'/[a-z]$//' )" && \ curl -fLO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" || \ @@ -121,8 +91,8 @@ RUN echo "Building OpenSSL" && \ env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make depend && \ env C_INCLUDE_PATH=/usr/local/musl/include/ make && \ - sudo make install && \ - sudo rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ + make install && \ + rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \ rm -r /tmp/* RUN echo "Building zlib" && \ @@ -130,7 +100,7 @@ RUN echo "Building zlib" && \ curl -fLO "http://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \ tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \ CC=musl-gcc ./configure --static --prefix=/usr/local/musl && \ - make && sudo make install && \ + make && make install && \ rm -r /tmp/* RUN echo "Building libpq" && \ @@ -138,10 +108,46 @@ RUN echo "Building libpq" && \ curl -fLO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \ tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \ CC=musl-gcc CPPFLAGS=-I/usr/local/musl/include LDFLAGS=-L/usr/local/musl/lib ./configure --with-openssl --without-readline --prefix=/usr/local/musl && \ - cd src/interfaces/libpq && make all-static-lib && sudo make install-lib-static && \ - cd ../../bin/pg_config && make && sudo make install && \ + cd src/interfaces/libpq && make all-static-lib && make install-lib-static && \ + cd ../../bin/pg_config && make && make install && \ rm -r /tmp/* +# (Please feel free to submit pull requests for musl-libc builds of other C +# libraries needed by the most popular and common Rust crates, to avoid +# everybody needing to build them manually.) + +# Install a `git credentials` helper for using GH_USER and GH_TOKEN to access +# private repositories if desired. We make sure this is configured for root, +# here, and for the `rust` user below. +ADD git-credential-ghtoken /usr/local/bin/ghtoken +RUN git config --global credential.https://github.com.helper ghtoken + +# Set up our path with all our binary directories, including those for the +# musl-gcc toolchain and for our Rust toolchain. +# +# We use the instructions at https://github.com/rust-lang/rustup/issues/2383 +# to install the rustup toolchain as root. +ENV RUSTUP_HOME=/opt/rust/rustup \ + PATH=/home/rust/.cargo/bin:/opt/rust/cargo/bin:/usr/local/musl/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +# Install our Rust toolchain and the `musl` target. We patch the +# command-line we pass to the installer so that it won't attempt to +# interact with the user or fool around with TTYs. We also set the default +# `--target` to musl so that our users don't need to keep overriding it +# manually. +RUN curl https://sh.rustup.rs -sSf | \ + env CARGO_HOME=/opt/rust/cargo \ + sh -s -- -y --default-toolchain $TOOLCHAIN --profile minimal --no-modify-path && \ + env CARGO_HOME=/opt/rust/cargo \ + rustup component add rustfmt && \ + env CARGO_HOME=/opt/rust/cargo \ + rustup component add clippy && \ + env CARGO_HOME=/opt/rust/cargo \ + rustup target add x86_64-unknown-linux-musl +ADD cargo-config.toml /opt/rust/cargo/config + +# Set up our environment variables so that we cross-compile using musl-libc by +# default. ENV X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_DIR=/usr/local/musl/ \ X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_STATIC=1 \ PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL=1 \ @@ -151,19 +157,28 @@ ENV X86_64_UNKNOWN_LINUX_MUSL_OPENSSL_DIR=/usr/local/musl/ \ LIBZ_SYS_STATIC=1 \ TARGET=musl -# (Please feel free to submit pull requests for musl-libc builds of other C -# libraries needed by the most popular and common Rust crates, to avoid -# everybody needing to build them manually.) - # Install some useful Rust tools from source. This will use the static linking # toolchain, but that should be OK. # # We include cargo-audit for compatibility with earlier versions of this image, -# but cargo-deny provides a super-set of cargo-audit's features. -RUN cargo install -f cargo-audit && \ - cargo install -f cargo-deb && \ - cargo install -f mdbook-graphviz && \ - rm -rf /home/rust/.cargo/registry/ +# but cargo-deny provides a superset of cargo-audit's features. +RUN env CARGO_HOME=/opt/rust/cargo cargo install -f cargo-audit && \ + env CARGO_HOME=/opt/rust/cargo cargo install -f cargo-deb && \ + env CARGO_HOME=/opt/rust/cargo cargo install -f mdbook-graphviz && \ + rm -rf /opt/rust/cargo/registry/ + +# Allow sudo without a password. +ADD sudoers /etc/sudoers.d/nopasswd + +# Run all further code as user `rust`, create our working directories, install +# our config file, and set up our credential helper. +# +# You should be able to switch back to `USER root` from another `Dockerfile` +# using this image if you need to do so. +USER rust +RUN mkdir -p /home/rust/libs /home/rust/src /home/rust/.cargo && \ + ln -s /opt/rust/cargo/config /home/rust/.cargo/config && \ + git config --global credential.https://github.com.helper ghtoken # Expect our source code to live in /home/rust/src. We'll run the build as # user `rust`, which will be uid 1000, gid 1000 outside the container. diff --git a/test-image b/test-image index c8b7989..f29ef40 100755 --- a/test-image +++ b/test-image @@ -15,32 +15,6 @@ docker run --rm rust-musl-builder-using-diesel echo "==== Verifying static linking" -# Make sure we can build a static executable. -docker run --rm ekidd/rust-musl-builder bash -c " -set -euo pipefail -export USER=rust -cargo new --vcs none --bin testme -cd testme - -echo -e '--- Test case for x86_64:' -cargo build -echo 'ldd says:' -if ldd target/x86_64-unknown-linux-musl/debug/testme; then - echo '[FAIL] Executable is not static!' 1>&2 - exit 1 -fi -echo -e '[PASS] x86_64 binary is statically linked.\n' - -echo -e '--- Test case for ARMhf:' -cargo build --target=armv7-unknown-linux-musleabihf -echo 'ldd says:' -if ldd target/armv7-unknown-linux-musleabihf/debug/testme; then - echo '[FAIL] Executable is not static!' 1>&2 - exit 1 -fi -echo -e '[PASS] ARMhf binary is statically linked.\n' -" - # Make sure we can build a static executable using `sqlx`. docker build -t rust-musl-builder-using-sqlx examples/using-sqlx docker run --rm rust-musl-builder-using-sqlx sh -c "