Merge pull request #73 from emk/experimental

Port to Ubuntu 18.04, add optional OpenSSL 1.1 support

.github/ISSUE_TEMPLATE/bug_report.md

@@ -19,8 +19,6 @@ Please include the steps you took, including the commands you ran in your termin
 
 **What did you hope to happen?**
 
-(A clear and concise description of what you expected to happen.)
-
 **Does `./test-image` work?**
 
 If you check out the repository on a Mac or Linux system and run `./build-image`, does it succeed or fail? This will build several known-good examples from the `rust-musl-builder/examples` directory.

.github/ISSUE_TEMPLATE/build_request.md

@@ -0,0 +1,12 @@
+---
+name: Build request
+about: Does the image need to be rebuilt?
+title: ''
+labels: bug
+assignees: emk
+
+---
+
+**Rust release:** (stable, X.Y.Z, beta, nightly-YYYY-MM-DD)
+
+**OpenSSL:** 1.0/1.1 (pick one)

.github/ISSUE_TEMPLATE/feature_request.md

@@ -4,10 +4,10 @@ about: I like hearing suggestions (but I don't always have time to implement the
   unfortunatey).
 title: ''
 labels: enhancement
-assignees: ''
+assignees: emk
 
 ---
 
 **How could this project be improved?**
 
-**If you're interested in implementing this feature yourself, would you like input or mentoring?**
+**If you're interested in implementing this feature yourself, how could I help you?**

Dockerfile

@@ -1,10 +1,13 @@
-# Use Debian 16.04 as the base for our Rust musl toolchain, because of
-# https://github.com/rust-lang/rust/issues/34978 (as of Rust 1.11).
-FROM ubuntu:16.04
+# Use Ubuntu 18.04 LTS as our base image.
+FROM ubuntu:18.04
 
 # The Rust toolchain to use when building our image.  Set by `hooks/build`.
 ARG TOOLCHAIN=stable
 
+# The OpenSSL version to use. We parameterize this because many Rust
+# projects will fail to build with 1.1.
+ARG OPENSSL_VERSION=1.0.2r
+
 # Make sure we have basic dev tools for building C libraries.  Our goal
 # here is to support the musl-libc builds and Cargo builds needed for a
 # large selection of the most popular crates.
@@ -26,14 +29,15 @@ RUN apt-get update && \
         libpq-dev \
         libsqlite-dev \
         libssl-dev \
+        linux-libc-dev \
         pkgconf \
         sudo \
         xutils-dev \
-        gcc-4.7-multilib-arm-linux-gnueabihf \
+        gcc-multilib-arm-linux-gnueabihf \
         && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
     useradd rust --user-group --create-home --shell /bin/bash --groups sudo && \
-    MDBOOK_VERSION=0.1.5 && \
+    MDBOOK_VERSION=0.2.1 && \
     curl -LO https://github.com/rust-lang-nursery/mdBook/releases/download/v$MDBOOK_VERSION/mdbook-v$MDBOOK_VERSION-x86_64-unknown-linux-musl.tar.gz && \
     tar xf mdbook-v$MDBOOK_VERSION-x86_64-unknown-linux-musl.tar.gz && \
     mv mdbook /usr/local/bin/ && \
@@ -70,34 +74,46 @@ ADD cargo-config.toml /home/rust/.cargo/config
 ADD git-credential-ghtoken /usr/local/bin
 RUN git config --global credential.https://github.com.helper ghtoken
 
-# Build a static library version of OpenSSL using musl-libc.  This is
-# needed by the popular Rust `hyper` crate.
+# Build a static library version of OpenSSL using musl-libc.  This is needed by
+# the popular Rust `hyper` crate.
+#
+# We point /usr/local/musl/include/linux at some Linux kernel headers (not
+# necessarily the right ones) in an effort to compile OpenSSL 1.1's "engine"
+# component. It's possible that this will cause bizarre and terrible things to
+# happen. There may be "sanitized" header
 RUN echo "Building OpenSSL" && \
+    ls /usr/include/linux && \
+    sudo mkdir -p /usr/local/musl/include && \
+    sudo ln -s /usr/include/linux /usr/local/musl/include/linux && \
+    sudo ln -s /usr/include/x86_64-linux-gnu/asm /usr/local/musl/include/asm && \
+    sudo ln -s /usr/include/asm-generic /usr/local/musl/include/asm-generic && \
     cd /tmp && \
-    OPENSSL_VERSION=1.0.2o && \
     curl -LO "https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz" && \
     tar xvzf "openssl-$OPENSSL_VERSION.tar.gz" && cd "openssl-$OPENSSL_VERSION" && \
-    env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl linux-x86_64 && \
+    env CC=musl-gcc ./Configure no-shared no-zlib -fPIC --prefix=/usr/local/musl -DOPENSSL_NO_SECURE_MEMORY linux-x86_64 && \
     env C_INCLUDE_PATH=/usr/local/musl/include/ make depend && \
-    make && sudo make install && \
-    \
-    echo "Building zlib" && \
+    env C_INCLUDE_PATH=/usr/local/musl/include/ make && \
+    sudo make install && \
+    sudo rm /usr/local/musl/include/linux /usr/local/musl/include/asm /usr/local/musl/include/asm-generic && \
+    rm -r /tmp/*
+
+RUN echo "Building zlib" && \
     cd /tmp && \
     ZLIB_VERSION=1.2.11 && \
     curl -LO "http://zlib.net/zlib-$ZLIB_VERSION.tar.gz" && \
     tar xzf "zlib-$ZLIB_VERSION.tar.gz" && cd "zlib-$ZLIB_VERSION" && \
     CC=musl-gcc ./configure --static --prefix=/usr/local/musl && \
     make && sudo make install && \
-    \
-    echo "Building libpq" && \
+    rm -r /tmp/*
+
+RUN echo "Building libpq" && \
     cd /tmp && \
-    POSTGRESQL_VERSION=9.6.8 && \
+    POSTGRESQL_VERSION=11.2 && \
     curl -LO "https://ftp.postgresql.org/pub/source/v$POSTGRESQL_VERSION/postgresql-$POSTGRESQL_VERSION.tar.gz" && \
     tar xzf "postgresql-$POSTGRESQL_VERSION.tar.gz" && cd "postgresql-$POSTGRESQL_VERSION" && \
     CC=musl-gcc CPPFLAGS=-I/usr/local/musl/include LDFLAGS=-L/usr/local/musl/lib ./configure --with-openssl --without-readline --prefix=/usr/local/musl && \
     cd src/interfaces/libpq && make all-static-lib && sudo make install-lib-static && \
     cd ../../bin/pg_config && make && sudo make install && \
-    \
     rm -r /tmp/*
 
 ENV OPENSSL_DIR=/usr/local/musl/ \

README.md

@@ -2,6 +2,9 @@
 
 [![Docker Image](https://img.shields.io/docker/pulls/ekidd/rust-musl-builder.svg?maxAge=2592000)](https://hub.docker.com/r/ekidd/rust-musl-builder/)
 
+**NOTE:** The underlying build image is now running Ubuntu 18.04 and newer
+versions of several libraries. Please report any problems!
+
 Do you want to compile a completely static Rust binary with no external dependencies?  If so, try:
 
 ```sh
@@ -11,18 +14,37 @@ rust-musl-builder cargo build --release
 
 This command assumes that `$(pwd)` is readable and writable by uid 1000, gid 1000. At the moment, it doesn't attempt to cache libraries between builds, so this is best reserved for making final release builds.
 
-To target ARM hard float (Raspberry Pi):
-
-```sh
-rust-musl-builder cargo build --target=armv7-unknown-linux-musleabihf --release
-```
-
-Binaries will be written to `target/$TARGET_ARCHITECTURE/release`. By default it targets `x86_64-unknown-linux-musl` unless specified with `--target`.
+For a more realistic example, see the `Dockerfile` for [examples/using-diesel](./examples/using-diesel).
 
 ## Deploying your Rust application
 
 With a bit of luck, you should be able to just copy your application binary from `target/x86_64-unknown-linux-musl/release`, and install it directly on any reasonably modern x86_64 Linux machine.  In particular, you should be able make static release binaries using TravisCI and GitHub, or you can copy your Rust application into an [Alpine Linux container][]. See below for details!
 
+## Available tags
+
+In general, we provide the following tagged Docker images:
+
+- `latest`, `stable`: Current stable Rust, with OpenSSL 1.0 (for now). We
+  try to update this fairly rapidly after every new stable release, and
+  after most point releases.
+- `X.Y.Z`: Specific versions of stable Rust.
+- `beta`: This usually gets updated every six weeks alongside the stable
+  release. It will usually not be updated for beta bugfix releases.
+- `nightly-YYYY-MM-DD`: Specific nightly releases. These should almost
+  always support `clippy`, `rls` and `rustfmt`, as verified using
+  [rustup components history][comp]. If you need a specific date for
+  compatibility with `tokio` or another popular library using unstable
+  Rust, please file an issue.
+- `stable-openssl11`: Current stable Rust, with OpenSSL 1.1.
+- `X.Y.Z-openssl11`: Specific versions of stable Rust, with OpenSSL 1.1.
+- `nightly-YYYY-MM-DD-openssl11`: Specific nightly releases, with OpenSSL
+  1.1.
+
+At a minimum, each of these images should be able to
+compile [examples/using-diesel](./examples/using-diesel).
+
+[comp]: https://rust-lang.github.io/rustup-components-history/index.html
+
 ## Caching builds
 
 You may be able to speed up build performance by adding the following `-v` commands to the `rust-musl-builder` alias:
@@ -46,7 +68,7 @@ rust-musl-builder sudo chown -R rust:rust \
 
 - The standard `musl-libc` libraries.
 - OpenSSL, which is needed by many Rust applications.
-- `libpq`, which is needed for applications that use `diesel` with PostgreSQL. Note that this may be broken under Rust 1.21.0 and later (see https://github.com/emk/rust-musl-builder/issues/27).
+- `libpq`, which is needed for applications that use `diesel` with PostgreSQL.
 - `libz`, which is needed by `libpq`.
 - SQLite3. See [examples/using-diesel](./examples/using-diesel/).
 
@@ -163,6 +185,18 @@ If you're using Docker crates which require specific C libraries to be installed
 
 If you need an especially common library, please feel free to submit a pull request adding it to the main `Dockerfile`!  We'd like to support popular Rust crates out of the box.
 
+## ARM support (experimental)
+
+To target ARM hard float (Raspberry Pi):
+
+```sh
+rust-musl-builder cargo build --target=armv7-unknown-linux-musleabihf --release
+```
+
+Binaries will be written to `target/$TARGET_ARCHITECTURE/release`. By default it targets `x86_64-unknown-linux-musl` unless specified with `--target`.
+
+This is missing many of the libraries used by the `x86_64` build, and it should probably be split out of the base image and given its own tags.
+
 ## Development notes
 
 After modifying the image, run `./test-image` to make sure that everything works.xs

cargo-config.toml

@@ -3,4 +3,4 @@
 target = "x86_64-unknown-linux-musl"
 
 [target.armv7-unknown-linux-musleabihf]
-linker = "arm-linux-gnueabihf-gcc-4.7"
+linker = "arm-linux-gnueabihf-gcc"

examples/using-diesel/Dockerfile

@@ -3,8 +3,12 @@
 # An example Dockerfile showing how to build a Rust executable using this
 # image, and deploy it with a tiny Alpine Linux container.
 
+# You can override this `--build-arg BASE_IMAGE=...` to use different
+# version of Rust or OpenSSL.
+ARG BASE_IMAGE=ekidd/rust-musl-builder:latest
+
 # Our first FROM statement declares the build environment.
-FROM ekidd/rust-musl-builder AS builder
+FROM ${BASE_IMAGE} AS builder
 
 # Add our source code.
 ADD . ./

hooks/build

@@ -1,14 +1,47 @@
 #!/bin/bash
+#
+# This script is responsible for calling `docker build` on behalf of GitHub. But
+# what it _really_ does is translate `DOCKER_TAG` into the builds args `TARGET`
+# and `OPENSSL_VERSION`.
 
 # Abort if anything goes wrong.
 set -euo pipefail
 
-# Always map the Docker tag `latest` to stable Rust.
-if [ "$DOCKER_TAG" == "latest" ]; then
+# Default to using OpenSSL 1.0 for a while longer, because 1.1 is
+# incompatible with the crates postgres 0.15 and openssl 0.9, which
+# are still widely used.
+OPENSSL_VERSION=1.0.2r
+
+# Pick an appropriate Docker tag
+case "$DOCKER_TAG" in
+  *-openssl11)
+    DOCKER_TAG_WITHOUT_OPENSSL="${DOCKER_TAG/-openssl11/}"
+    OPENSSL_VERSION=1.1.1b
+    ;;
+  *)
+    DOCKER_TAG_WITHOUT_OPENSSL="$DOCKER_TAG"
+    ;;
+esac
+
+# Decide what Rust toolchain to use.
+case "$DOCKER_TAG_WITHOUT_OPENSSL" in
+  # Always map the Docker tags `latest` and `experimental` to stable Rust.
+  latest)
     TOOLCHAIN=stable
-else
-  TOOLCHAIN="$DOCKER_TAG"
-fi
+    ;;
+  # Strip `experimental-` from other `experimental-*` tags.
+  experimental-*)
+    TOOLCHAIN="${DOCKER_TAG_WITHOUT_OPENSSL/experimental-/}"
+    ;;
+  # Pass all other tags through.
+  *)
+    TOOLCHAIN="$DOCKER_TAG_WITHOUT_OPENSSL"
+    ;;
+esac
 
 # Run the build.
-docker build --build-arg TOOLCHAIN="$TOOLCHAIN" -t "$IMAGE_NAME" .
+docker build \
+  --build-arg TOOLCHAIN="$TOOLCHAIN" \
+  --build-arg OPENSSL_VERSION="$OPENSSL_VERSION" \
+  -t "$IMAGE_NAME" \
+  .

hooks/test

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Abort if anything goes wrong.
+set -euo pipefail
+
+# Make sure we can build some of our more important test images.
+for EXAMPLE in using-diesel; do
+    docker build \
+        --build-arg BASE_IMAGE="$IMAGE_NAME" \
+        -t rust-musl-builder-"$EXAMPLE" \
+        examples/"$EXAMPLE"
+done