release: Produce static binaries (#2057)

This change adds release targets for static binaries (using musl).
Expected checksec outputs are split per platform, to account
for cross-compiled MUSL binaries.

With this change, we retain debug symbols in an separate file. So
they can be loaded in a debugger.

This change bumps the devcontainer to v38 to include just-cargo fixes.

.checksec/amd64-gnu.json

@@ -1,10 +1,8 @@
 {
   "canary": "yes",
-  "fortify_source": "no",
   "nx": "yes",
   "pie": "yes",
   "relro": "full",
   "rpath": "no",
-  "runpath": "no",
+  "runpath": "no"
-  "symbols": "no"
 }

.checksec/amd64-musl.json

@@ -0,0 +1,8 @@
+{
+  "canary": "yes",
+  "nx": "yes",
+  "pie": "yes",
+  "relro": "full",
+  "rpath": "no",
+  "runpath": "no"
+}

.checksec/arm-gnu.json

@@ -0,0 +1,8 @@
+{
+  "canary": "yes",
+  "nx": "yes",
+  "pie": "yes",
+  "relro": "full",
+  "rpath": "no",
+  "runpath": "no"
+}

.checksec/arm-musl.json

@@ -0,0 +1,8 @@
+{
+  "canary": "yes",
+  "nx": "yes",
+  "pie": "no",
+  "relro": "partial",
+  "rpath": "no",
+  "runpath": "no"
+}

.checksec/arm64-gnu.json

@@ -0,0 +1,8 @@
+{
+  "canary": "yes",
+  "nx": "yes",
+  "pie": "yes",
+  "relro": "full",
+  "rpath": "no",
+  "runpath": "no"
+}

.checksec/arm64-musl.json

@@ -0,0 +1,8 @@
+{
+  "canary": "yes",
+  "nx": "yes",
+  "pie": "no",
+  "relro": "partial",
+  "rpath": "no",
+  "runpath": "no"
+}

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
 	"name": "linkerd2-proxy",
-	"image": "ghcr.io/linkerd/dev:v37",
+	"image": "ghcr.io/linkerd/dev:v38",
 	"extensions": [
 		"DavidAnson.vscode-markdownlint",
 		"kokakiwi.vscode-just",

.github/actions/list-changed-crates/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/linkerd/dev:v37-rust
+FROM ghcr.io/linkerd/dev:v38-rust
 RUN apt-get update && apt-get install -y --no-install-recommends jq \
     && rm -rf /var/lib/apt/lists/*
 COPY entrypoint.sh /

.github/workflows/beta.yml

@@ -22,7 +22,7 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     timeout-minutes: 20
     continue-on-error: true
     steps:

.github/workflows/check-all.yml

@@ -22,7 +22,7 @@ jobs:
   check-all:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: just fetch

.github/workflows/check-each.yml

@@ -49,7 +49,7 @@ jobs:
     needs: list-changed-crates
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     strategy:
       matrix:
         crate: ${{ fromJson(needs.list-changed-crates.outputs.crates) }}

.github/workflows/coverage.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     container:
-      image: docker://ghcr.io/linkerd/dev:v37-rust
+      image: docker://ghcr.io/linkerd/dev:v38-rust
       options: --security-opt seccomp=unconfined # 🤷
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8

.github/workflows/deps.yml

@@ -46,7 +46,7 @@ jobs:
   deprecated:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: just fetch

.github/workflows/docker.yml

@@ -15,9 +15,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     timeout-minutes: 20
-    env:
-      DOCKER_BUILDKIT: "1"
     steps:
-      - uses: extractions/setup-just@95b912dc5d3ed106a72907f2f9b91e76d60bdb76
+      - uses: linkerd/dev/actions/setup-tools@v38
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: just docker

.github/workflows/integration.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     timeout-minutes: 20
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: just fetch

.github/workflows/lint.yml

@@ -21,7 +21,7 @@ jobs:
   clippy:
     timeout-minutes: 10
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: just fetch
@@ -30,7 +30,7 @@ jobs:
   fmt:
     timeout-minutes: 10
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: just check-fmt
@@ -38,7 +38,7 @@ jobs:
   docs:
     timeout-minutes: 10
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: just fetch

.github/workflows/nightly.yml

@@ -22,7 +22,7 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     timeout-minutes: 20
     continue-on-error: true
     steps:

.github/workflows/release.yml

@@ -52,40 +52,20 @@ jobs:
     strategy:
       matrix:
         arch: [amd64, arm64, arm]
+        libc: [gnu, musl]
+
     # If we're not actually building on a release tag, don't short-circuit on
     # errors. This helps us know whether a failure is platform-specific.
     continue-on-error: ${{ !needs.meta.outputs.publish }}
-    name: Package (${{ matrix.arch }})
     runs-on: ubuntu-latest
     timeout-minutes: 40
-    container: docker://ghcr.io/linkerd/dev:v37-rust
+    container: docker://ghcr.io/linkerd/dev:v38-rust-musl
     steps:
-      - if: matrix.arch == 'arm64'
-        run: |
-          rustup target add aarch64-unknown-linux-gnu
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            binutils-aarch64-linux-gnu \
-            g++-aarch64-linux-gnu \
-            gcc-aarch64-linux-gnu \
-            libc6-dev-arm64-cross
-          echo CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc >> "$GITHUB_ENV"
-
-      - if: matrix.arch == 'arm'
-        run: |
-          rustup target add armv7-unknown-linux-gnueabihf
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            binutils-arm-linux-gnueabihf \
-            g++-arm-linux-gnueabihf \
-            gcc-arm-linux-gnueabihf \
-            libc6-dev-armhf-cross
-          echo CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc >> "$GITHUB_ENV"
-
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: just fetch
-      - name: Run just package
+      - run: just arch=${{ matrix.arch }} libc=${{ matrix.libc }} rustup
-        run: just profile=release package_arch=${{ matrix.arch }} package_version=${{ needs.meta.outputs.version }} package
+      - run: just arch=${{ matrix.arch }} libc=${{ matrix.libc }} profile=release build
+      - run: just arch=${{ matrix.arch }} libc=${{ matrix.libc }} profile=release package_version=${{ needs.meta.outputs.version }} package
       - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
         with:
           name: ${{ matrix.arch }}-artifacts

.github/workflows/test.yml

@@ -24,7 +24,7 @@ jobs:
   meshtls:
     timeout-minutes: 10
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: just fetch
@@ -42,7 +42,7 @@ jobs:
   unit:
     timeout-minutes: 10
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: just fetch

.github/workflows/toolchain.yml

@@ -14,7 +14,7 @@ permissions:
 jobs:
   devcontainer:
     runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
       - run: |
@@ -24,22 +24,6 @@ jobs:
             exit 1
           fi
 
-  dockerfiles:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
-      - run: |
-          VERSION="$(cat rust-toolchain)"
-          ex=0
-          while IFS= read -r file ; do
-            versions=$(sed -nE 's/^ARG RUST_VERSION=([^ ]+)/\1/p' "$file")
-            for mismatch in $(echo "$versions" | grep -vF "$VERSION" || true) ; do
-              echo "::error file=$file::$file uses incorrect rust version(s): $mismatch"
-              ex=$((ex + 1))
-            done
-          done < <(find . -name Dockerfile)
-          exit $ex
-
   workflows:
     runs-on: ubuntu-latest
     steps:

Cargo.toml

@@ -64,7 +64,7 @@ members = [
     "linkerd/transport-metrics",
     "linkerd2-proxy",
     "opencensus-proto",
-    "tools"
+    "tools",
 ]
 
 # Debug symbols end up chewing up several GB of disk space, so better to just

Dockerfile

@@ -16,50 +16,39 @@
 #
 #     :; docker buildx build . --load
 
-ARG RUST_VERSION=1.64.0
+ARG RUST_IMAGE=ghcr.io/linkerd/dev:v38-rust
-ARG RUST_IMAGE=rust:${RUST_VERSION}-bullseye
 
 # Use an arbitrary ~recent edge release image to get the proxy
 # identity-initializing and linkerd-await wrappers.
-ARG RUNTIME_IMAGE=ghcr.io/linkerd/proxy:edge-22.2.1
+ARG RUNTIME_IMAGE=ghcr.io/linkerd/proxy:edge-22.12.1
 
 # Build the proxy, leveraging (new, experimental) cache mounting.
 #
 # See: https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md#run---mounttypecache
-FROM $RUST_IMAGE as build
+FROM --platform=$BUILDPLATFORM $RUST_IMAGE as build
 
-# When set, causes the proxy to be compiled in development mode.
+ARG PROXY_FEATURES=""
-ARG PROXY_UNOPTIMIZED
+RUN apt-get update && \
-
+    apt-get install -y time && \
-# Controls what features are enabled in the proxy.
+    if [[ "$PROXY_FEATURES" =~ .*meshtls-boring.* ]] ; then \
-ARG PROXY_FEATURES="multicore,meshtls-rustls"
+      apt-get install -y golang ; \
-
+    fi && \
-RUN --mount=type=cache,target=/var/lib/apt/lists \
+    rm -rf /var/lib/apt/lists/*
-    --mount=type=cache,target=/var/tmp \
-  apt update && apt install -y time
-
-RUN --mount=type=cache,target=/var/lib/apt/lists \
-    --mount=type=cache,target=/var/tmp \
-  if $(echo "$PROXY_FEATURES" | grep "meshtls-boring" >/dev/null); then \
-    apt install -y cmake clang golang ; \
-  fi
 
 WORKDIR /usr/src/linkerd2-proxy
 COPY . .
-RUN --mount=type=cache,target=target \
+RUN --mount=type=cache,id=cargo,target=/usr/local/cargo/registry \
-    --mount=type=cache,from=rust:1.64.0-bullseye,source=/usr/local/cargo,target=/usr/local/cargo \
+    just fetch
-  mkdir -p /out && \
+ARG TARGETARCH="amd64"
-  if [ -n "$PROXY_UNOPTIMIZED" ]; then \
+ARG PROFILE="release"
-    (cd linkerd2-proxy && /usr/bin/time -v cargo build --locked --no-default-features --features="$PROXY_FEATURES") && \
+RUN --mount=type=cache,id=target,target=target \
-    mv target/debug/linkerd2-proxy /out/linkerd2-proxy ; \
+    --mount=type=cache,id=cargo,target=/usr/local/cargo/registry \
-  else \
+    just arch=$TARGETARCH features=$PROXY_FEATURES profile=$PROFILE build && \
-    (cd linkerd2-proxy && /usr/bin/time -v cargo build --locked --no-default-features --features="$PROXY_FEATURES" --release) && \
+    bin=$(just --evaluate profile="$PROFILE" _target_bin) ; \
-    mv target/release/linkerd2-proxy /out/linkerd2-proxy ; \
+    mkdir -p /out && mv $bin /out/linkerd2-proxy
-  fi
 
 ## Install the proxy binary into the base runtime image.
 FROM $RUNTIME_IMAGE as runtime
-
 WORKDIR /linkerd
 COPY --from=build /out/linkerd2-proxy /usr/lib/linkerd/linkerd2-proxy
 ENV LINKERD2_PROXY_LOG=warn,linkerd=info

justfile

@@ -12,7 +12,6 @@ export PROTOC_NO_VENDOR := "1"
 # By default we compile in development mode mode because it's faster.
 profile := if env_var_or_default("RELEASE", "") == "" { "debug" } else { "release" }
 toolchain := ""
-_cargo := "just-cargo profile=" + profile + " toolchain=" + toolchain
 
 features := ""
 
@@ -25,25 +24,28 @@ docker_tag := `git rev-parse --abbrev-ref HEAD | sed 's|/|.|'` + "." + `git rev-
 docker_image := docker_repo + ":" + docker_tag
 
 # The architecture name to use for packages. Either 'amd64', 'arm64', or 'arm'.
-package_arch := "amd64"
+arch := "amd64"
 
-# If a `package_arch` is specified, then we change the default cargo `--target`
+libc := 'gnu'
+
+# If a `arch` is specified, then we change the default cargo `--target`
 # to support cross-compilation. Otherwise, we use `rustup` to find the default.
-cargo_target := if package_arch == "arm64" {
+_target := if arch == 'amd64' {
-        "aarch64-unknown-linux-gnu"
+        "x86_64-unknown-linux-" + libc
-    } else if package_arch == "arm" {
+    } else if arch == "arm64" {
-        "armv7-unknown-linux-gnueabihf"
+        "aarch64-unknown-linux-" + libc
+    } else if arch == "arm" {
+        "armv7-unknown-linux-" + libc + "eabihf"
     } else {
-        `rustup show | sed -n 's/^Default host: \(.*\)/\1/p'`
+        error("unsupported arch=" + arch)
     }
 
-# Support cross-compilation when `package_arch` changes.
+_cargo := 'just-cargo profile=' + profile + ' target=' + _target + ' toolchain=' + toolchain
-strip := if package_arch == "arm64" { "aarch64-linux-gnu-strip" } else if package_arch == "arm" { "arm-linux-gnueabihf-strip" } else { "strip" }
 
-target_dir := join("target", cargo_target, profile)
+_target_dir := "target" / _target / profile
-target_bin := join(target_dir, "linkerd2-proxy")
+_target_bin := _target_dir / "linkerd2-proxy"
-package_name := "linkerd2-proxy-" + package_version + "-" + package_arch
+_package_name := "linkerd2-proxy-" + package_version + "-" + arch + if libc == 'musl' { '-static' } else { '' }
-package_dir := join("target/package", package_name)
+_package_dir := "target/package" / _package_name
 shasum := "shasum -a 256"
 
 _features := if features == "all" {
@@ -56,6 +58,9 @@ _features := if features == "all" {
 # Recipes
 #
 
+rustup:
+    @{{ _cargo }} _target-installed
+
 # Run all lints
 lint: sh-lint md-lint clippy doc action-lint action-dev-check
 
@@ -106,27 +111,40 @@ test-dir dir *flags:
     cd {{ dir }} && {{ _cargo }} test --frozen {{ _features }} {{ flags }}
 
 # Build the proxy
-build:
+build: && checksec _strip
-    @{{ _cargo }} build --frozen --package=linkerd2-proxy --target={{ cargo_target }} {{ _features }}
+    @rm -f {{ _target_bin }} {{ _target_bin }}.dbg
+    @{{ _cargo }} build --frozen --package=linkerd2-proxy {{ _features }}
 
-_package_bin := package_dir / "bin" / "linkerd2-proxy"
+_strip:
+    {{ _objcopy }} --only-keep-debug {{ _target_bin }} {{ _target_bin }}.dbg
+    {{ _objcopy }} --strip-unneeded {{ _target_bin }}
+    {{ _objcopy }} --add-gnu-debuglink={{ _target_bin }}.dbg {{ _target_bin }}
+
+_package_bin := _package_dir / "bin" / "linkerd2-proxy"
+
+# XXX {aarch64,arm}-musl builds do not enable PIE, so we use target-specific
+# files to document those differences.
+_expected_checksec := '.checksec' / arch + '-' + libc + '.json'
+
+# Check the security properties of the proxy binary.
+checksec:
+    checksec --output=json --file='{{ _target_bin }}' \
+        | jq '.' | tee /dev/stderr \
+        | jq -S '.[] | del(."fortify_source") | del(."fortify-able") | del(.fortified) | del(.symbols)' \
+        | diff -u {{ _expected_checksec }} - >&2
+
+_objcopy := 'llvm-objcopy-' + `just-cargo --evaluate _llvm-version`
 
 # Build a package (i.e. for a release)
 package: build
-    mkdir -p {{ package_dir }}/bin
+    @mkdir -p {{ _package_dir }}/bin
-    cp LICENSE {{ package_dir }}/
+    cp LICENSE {{ _package_dir }}/
-    cp {{ target_dir }}/linkerd2-proxy {{ _package_bin }}
+    cp {{ _target_bin }} {{ _target_bin }}.dbg {{ _package_dir }}/
-    {{ strip }} {{ _package_bin }}
+    tar -czvf target/package/{{ _package_name }}.tar.gz  -C target/package {{ _package_name }} >/dev/null
-    checksec --output=json --file='{{ _package_bin }}' \
+    cd target/package && ({{ shasum }} {{ _package_name }}.tar.gz | tee {{ _package_name }}.txt)
-        | jq '.["{{ _package_bin }}"] | del(."fortify-able") | del(.fortified)' \
+    @rm -rf {{ _package_dir }}
-        > target/package/{{ package_name }}-checksec.json
+    @du -h target/package/{{ _package_name }}.tar.gz
-    jq -S '.'  target/package/{{ package_name }}-checksec.json \
+    @tar tzvf target/package/{{ _package_name }}.tar.gz
-        | diff -u .checksec-expected.json - >&2
-    cd target/package \
-        && (tar -czvf {{ package_name }}.tar.gz {{ package_name }} >/dev/null) \
-        && ({{ shasum }} {{ package_name }}.tar.gz > {{ package_name }}.txt)
-    @rm -rf {{ package_dir }}
-    @du -h target/package/{{ package_name }}*
 
 # Build all of the fuzzers (SLOW).
 fuzzers:
@@ -141,18 +159,19 @@ fuzzers:
         echo "cd $dir && {{ _cargo }} fuzz build"
         (
             cd $dir
-            @{{ _cargo }} fuzz build --target={{ cargo_target }} \
+            @{{ _cargo }} fuzz build \
                 {{ if profile == "release" { "--release" } else { "" } }}
         )
     done
 
 # Build a docker image (FOR TESTING ONLY)
-docker mode='load':
+docker *args='--output=type=docker':
     docker buildx build . \
+        --pull \
         --tag={{ docker_image }} \
-        {{ if profile != 'release' { "--build-arg PROXY_UNOPTIMIZED=1" } else { "" } }} \
+        --build-arg PROFILE='{{ profile }}' \
         {{ if features != "" { "--build-arg PROXY_FEATURES=" + features } else { "" } }} \
-        {{ if mode == 'push' { "--push" } else { "--load" } }}
+        {{ args }}
 
 # Lints all shell scripts in the repo.
 sh-lint: