docker: Use buildkit for caching (#472)

Our docker builds do not permit caching of dependencies or intermediate
build artifacts. However, Docker's new (experimental) buildkit features
add this functionality. I've been using this configuration locally for
some time, and it seems generaly useful enough to promote (especially
since this Dockerfile is only intended for development).

This change also untroduces the `PROXY_FEATURES` build-arg so that the
Dockerfile can be used to support profiling builds.

Furthermore, the `update-rust-versions.sh` script has been updated to
check versions and be more permissive about how it replaces versions in
the Dockerfile.

The docker CI build has been disabled until GitHub Actions has support
for this Dockerfile format.

.github/workflows/docker.yml

@@ -6,14 +6,14 @@ name: Docker
 on:
   push:
     branches:
-    - master
+      - master
     paths:
-    - 'Cargo.lock'
-    - 'Dockerfile'
+      - "Cargo.lock"
+      - "Dockerfile"
   pull_request:
     paths:
-    - 'Cargo.lock'
-    - 'Dockerfile'
+      - "Cargo.lock"
+      - "Dockerfile"
 
 env:
   DOCKER_UNOPTIMIZED: "1"
@@ -22,43 +22,43 @@ jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
+      - uses: actions/checkout@v1
 
-    # Create a build image on a Linkerd build host.
-    - name: Setup (Origin)
-      if: '!github.event.pull_request.head.repo.fork'
-      run: |
-        mkdir -p ~/.ssh
-        # Create an identity file and protect before writing contents to it.
-        touch ~/.ssh/id && chmod 600 ~/.ssh/id
-        echo "${{ secrets.DOCKER_PRIVATE_KEY }}" >~/.ssh/id
-        # Use well-known public keys for the host to prevent middlemen.
-        echo "${{ secrets.DOCKER_KNOWN_HOSTS }}" >~/.ssh/known_hosts
-        # Configure host with ServerAliveInterval to ensure that the client
-        # stays alive even when the server is busy emitting nothing.
-        # ServerAliveCountMax ensures that server responds to these pings
-        # within ~5 minutes.
-        (
-          echo "Host linkerd-docker"
-          echo "    User github"
-          echo "    Hostname ${{ secrets.DOCKER_ADDRESS }}"
-          echo "    IdentityFile ~/.ssh/id"
-          echo "    BatchMode yes"
-          echo "    ServerAliveInterval 60"
-          echo "    ServerAliveCountMax 5"
-        ) >~/.ssh/config
-        # Confirm that the SSH configuration works.
-        ssh linkerd-docker docker version
+      # Create a build image on a Linkerd build host.
+      - name: Setup (Origin)
+        if: "!github.event.pull_request.head.repo.fork"
+        run: |
+          mkdir -p ~/.ssh
+          # Create an identity file and protect before writing contents to it.
+          touch ~/.ssh/id && chmod 600 ~/.ssh/id
+          echo "${{ secrets.DOCKER_PRIVATE_KEY }}" >~/.ssh/id
+          # Use well-known public keys for the host to prevent middlemen.
+          echo "${{ secrets.DOCKER_KNOWN_HOSTS }}" >~/.ssh/known_hosts
+          # Configure host with ServerAliveInterval to ensure that the client
+          # stays alive even when the server is busy emitting nothing.
+          # ServerAliveCountMax ensures that server responds to these pings
+          # within ~5 minutes.
+          (
+            echo "Host linkerd-docker"
+            echo "    User github"
+            echo "    Hostname ${{ secrets.DOCKER_ADDRESS }}"
+            echo "    IdentityFile ~/.ssh/id"
+            echo "    BatchMode yes"
+            echo "    ServerAliveInterval 60"
+            echo "    ServerAliveCountMax 5"
+          ) >~/.ssh/config
+          # Confirm that the SSH configuration works.
+          ssh linkerd-docker docker version
 
-    - name: Docker (Origin)
-      if: '!github.event.pull_request.head.repo.fork'
-      env:
-        DOCKER_HOST: "ssh://linkerd-docker"
-      run: |
-        export DOCKER_TAG="proxy-ci:$(dd bs=64 count=1 if=/dev/urandom status=none | tr -dc 'a-zA-Z0-9')"
-        make docker
-        docker image rm -f "$DOCKER_TAG"
+      - name: Docker (Origin)
+        if: "!github.event.pull_request.head.repo.fork"
+        env:
+          DOCKER_HOST: "ssh://linkerd-docker"
+        run: |
+          export DOCKER_TAG="proxy-ci:$(dd bs=64 count=1 if=/dev/urandom status=none | tr -dc 'a-zA-Z0-9')"
+          make docker
+          docker image rm -f "$DOCKER_TAG"
 
-    - name: Docker (Fork)
-      if: github.event.pull_request.head.repo.fork
-      run: make docker
+      - name: Docker (Fork)
+        if: github.event.pull_request.head.repo.fork
+        run: make docker

Dockerfile

@@ -1,36 +1,56 @@
+# syntax=docker/dockerfile:experimental
+
 # Proxy build and runtime
 #
 # This is intended **DEVELOPMENT ONLY**, i.e. so that proxy developers can
 # easily test the proxy in the context of the larger `linkerd2` project.
 #
-# When PROXY_UNOPTIMIZED is set and not empty, unoptimized rust artifacts are produced.
-# This reduces build time and produces binaries with debug symbols, at the expense of
-# runtime performance.
+# This Dockerfile requires expirmental features to be enabled in both the
+# Docker client and daemon. You MUST use buildkit to build this image. The
+# simplest way to do this is to set the `DOCKER_BUILDKIT` environment variable:
+#
+#     :; DOCKER_BUILDKIT=1 docker build .
+#
+# Alternatively, you can use `buildx`, which supports more complicated build
+# configurations:
+#
+#     :; docker buildx build . --load
 
-# rather than updating this manually, run update-rust-version.sh
+# Please make changes via update-rust-version.sh
 ARG RUST_IMAGE=rust:1.41.0-buster
-ARG RUNTIME_IMAGE=gcr.io/linkerd-io/proxy:edge-20.2.2
-ARG PROXY_UNOPTIMIZED
 
-## Builds the proxy as incrementally as possible.
+# Use an arbitrary ~recent edge release image to get the proxy
+# identity-initializing wrapper.
+ARG RUNTIME_IMAGE=gcr.io/linkerd-io/proxy:edge-20.4.1
+
+# Build the proxy, leveraging (new, experimental) cache mounting.
+#
+# See: https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md#run---mounttypecache
 FROM $RUST_IMAGE as build
 
+# When set, causes the proxy to be compiled in development mode.
+ARG PROXY_UNOPTIMIZED
+
+# Controls what features are enabled in the proxy. This is typically empty but
+# may be set to `mock-orig-dst` for profiling builds.
+ARG PROXY_FEATURES
+
 WORKDIR /usr/src/linkerd2-proxy
-
 COPY . .
-RUN cargo fetch --locked
-RUN if [ -n "$PROXY_UNOPTIMIZED" ]; \
-    then \
-    cargo build -p linkerd2-proxy --bin linkerd2-proxy --frozen && \
-    mv target/debug/linkerd2-proxy target/linkerd2-proxy ; \
+RUN --mount=type=cache,target=target \
+    --mount=type=cache,from=rust:1.41.0-buster,source=/usr/local/cargo,target=/usr/local/cargo \
+    mkdir -p /out && \
+    if [ -n "$PROXY_UNOPTIMIZED" ]; then \
+      (cd linkerd2-proxy && cargo build --locked --features="$PROXY_FEATURES") && \
+      mv target/debug/linkerd2-proxy /out/linkerd2-proxy ; \
     else \
-    cargo build -p linkerd2-proxy --bin linkerd2-proxy --frozen --release && \
-    mv target/release/linkerd2-proxy target/linkerd2-proxy ; \
+      (cd linkerd2-proxy && cargo build --locked --release --features="$PROXY_FEATURES") && \
+      mv target/release/linkerd2-proxy /out/linkerd2-proxy ; \
     fi
 
-
 ## Install the proxy binary into the base runtime image.
 FROM $RUNTIME_IMAGE as runtime
 WORKDIR /linkerd
-COPY --from=build /usr/src/linkerd2-proxy/target/linkerd2-proxy /usr/lib/linkerd/linkerd2-proxy
-ENV LINKERD2_PROXY_LOG=warn,linkerd2_proxy=info
+COPY --from=build /out/linkerd2-proxy /usr/lib/linkerd/linkerd2-proxy
+ENV LINKERD2_PROXY_LOG=warn,linkerd=info
+# Inherits the ENTRYPOINT from the runtime image.

Makefile

@@ -105,7 +105,7 @@ clean-profile:
 
 .PHONY: docker
 docker: Dockerfile Cargo.lock
-	$(DOCKER_BUILD) .
+	DOCKER_BUILDKIT=1 $(DOCKER_BUILD) .
 
 .PHONY: all
 all: build test

update-rust-version.sh

@@ -7,10 +7,14 @@ if [ $# -ne 1 ]; then
     exit 64
 fi
 
-VERSION=$1
+VERSION="$1"
+if ! echo "$VERSION" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' ; then
+    echo "Expected M.N.P; got '$VERSION'" >&2
+    exit 64
+fi
 
 echo "$VERSION" > rust-toolchain
-sed -i'' -e "s/RUST_IMAGE=.*/RUST_IMAGE=rust:$VERSION-buster/" Dockerfile
+sed -i'' -Ee "s/rust:[0-9]+\.[0-9]+\.[0-9]+/rust:$VERSION/" Dockerfile
 
 find .github -name \*.yml \
-    -exec sed -i'' -e "s|docker://rust:.*|docker://rust:$VERSION-buster|" '{}' \;
+    -exec sed -i'' -Ee "s|docker://rust:[0-9]+\.[0-9]+\.[0-9]+|docker://rust:$VERSION|" '{}' \;