release: Produce static binaries (#2057)

This change adds release targets for static binaries (using musl). Expected checksec outputs are split per platform, to account for cross-compiled MUSL binaries. With this change, we retain debug symbols in an separate file. So they can be loaded in a debugger. This change bumps the devcontainer to v38 to include just-cargo fixes.
2022-12-12 16:10:11 -08:00 · 2022-12-12 16:10:11 -08:00 · adf06c492d
parent 194ca211b5
commit adf06c492d
23 changed files with 137 additions and 129 deletions
--- a/.checksec/amd64-gnu.json
+++ b/.checksec/amd64-gnu.json
@ -1,10 +1,8 @@
 {
  "canary": "yes",
-  "fortify_source": "no",
  "nx": "yes",
  "pie": "yes",
  "relro": "full",
  "rpath": "no",
-  "runpath": "no",
-  "symbols": "no"
+  "runpath": "no"
 }
--- a/.checksec/amd64-musl.json
+++ b/.checksec/amd64-musl.json
@ -0,0 +1,8 @@
+{
+  "canary": "yes",
+  "nx": "yes",
+  "pie": "yes",
+  "relro": "full",
+  "rpath": "no",
+  "runpath": "no"
+}
--- a/.checksec/arm-gnu.json
+++ b/.checksec/arm-gnu.json
@ -0,0 +1,8 @@
+{
+  "canary": "yes",
+  "nx": "yes",
+  "pie": "yes",
+  "relro": "full",
+  "rpath": "no",
+  "runpath": "no"
+}
--- a/.checksec/arm-musl.json
+++ b/.checksec/arm-musl.json
@ -0,0 +1,8 @@
+{
+  "canary": "yes",
+  "nx": "yes",
+  "pie": "no",
+  "relro": "partial",
+  "rpath": "no",
+  "runpath": "no"
+}
--- a/.checksec/arm64-gnu.json
+++ b/.checksec/arm64-gnu.json
@ -0,0 +1,8 @@
+{
+  "canary": "yes",
+  "nx": "yes",
+  "pie": "yes",
+  "relro": "full",
+  "rpath": "no",
+  "runpath": "no"
+}
--- a/.checksec/arm64-musl.json
+++ b/.checksec/arm64-musl.json
@ -0,0 +1,8 @@
+{
+  "canary": "yes",
+  "nx": "yes",
+  "pie": "no",
+  "relro": "partial",
+  "rpath": "no",
+  "runpath": "no"
+}
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@ -1,6 +1,6 @@
 {
 	"name": "linkerd2-proxy",
-	"image": "ghcr.io/linkerd/dev:v37",
+	"image": "ghcr.io/linkerd/dev:v38",
 	"extensions": [
 		"DavidAnson.vscode-markdownlint",
 		"kokakiwi.vscode-just",
--- a/.github/actions/list-changed-crates/Dockerfile
+++ b/.github/actions/list-changed-crates/Dockerfile
@ -1,4 +1,4 @@
-FROM ghcr.io/linkerd/dev:v37-rust
+FROM ghcr.io/linkerd/dev:v38-rust
 RUN apt-get update && apt-get install -y --no-install-recommends jq \
    && rm -rf /var/lib/apt/lists/*
 COPY entrypoint.sh /
--- a/.github/workflows/beta.yml
+++ b/.github/workflows/beta.yml
@ -22,7 +22,7 @@ permissions:
 jobs:
  build:
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    timeout-minutes: 20
    continue-on-error: true
    steps:
--- a/.github/workflows/check-all.yml
+++ b/.github/workflows/check-all.yml
@ -22,7 +22,7 @@ jobs:
  check-all:
    timeout-minutes: 20
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    steps:
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: just fetch
--- a/.github/workflows/check-each.yml
+++ b/.github/workflows/check-each.yml
@ -49,7 +49,7 @@ jobs:
    needs: list-changed-crates
    timeout-minutes: 20
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    strategy:
      matrix:
        crate: ${{ fromJson(needs.list-changed-crates.outputs.crates) }}
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@ -23,7 +23,7 @@ jobs:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    container:
-      image: docker://ghcr.io/linkerd/dev:v37-rust
+      image: docker://ghcr.io/linkerd/dev:v38-rust
      options: --security-opt seccomp=unconfined # 🤷
    steps:
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
--- a/.github/workflows/deps.yml
+++ b/.github/workflows/deps.yml
@ -46,7 +46,7 @@ jobs:
  deprecated:
    timeout-minutes: 20
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    steps:
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: just fetch
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@ -15,9 +15,7 @@ jobs:
  build:
    runs-on: ubuntu-latest
    timeout-minutes: 20
-    env:
-      DOCKER_BUILDKIT: "1"
    steps:
-      - uses: extractions/setup-just@95b912dc5d3ed106a72907f2f9b91e76d60bdb76
+      - uses: linkerd/dev/actions/setup-tools@v38
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: just docker
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@ -26,7 +26,7 @@ jobs:
  test:
    timeout-minutes: 20
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    steps:
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: just fetch
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@ -21,7 +21,7 @@ jobs:
  clippy:
    timeout-minutes: 10
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    steps:
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: just fetch
@ -30,7 +30,7 @@ jobs:
  fmt:
    timeout-minutes: 10
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    steps:
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: just check-fmt
@ -38,7 +38,7 @@ jobs:
  docs:
    timeout-minutes: 10
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    steps:
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: just fetch
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@ -22,7 +22,7 @@ permissions:
 jobs:
  build:
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    timeout-minutes: 20
    continue-on-error: true
    steps:
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -52,40 +52,20 @@ jobs:
    strategy:
      matrix:
        arch: [amd64, arm64, arm]
+        libc: [gnu, musl]
+
    # If we're not actually building on a release tag, don't short-circuit on
    # errors. This helps us know whether a failure is platform-specific.
    continue-on-error: ${{ !needs.meta.outputs.publish }}
-    name: Package (${{ matrix.arch }})
    runs-on: ubuntu-latest
    timeout-minutes: 40
-    container: docker://ghcr.io/linkerd/dev:v37-rust
+    container: docker://ghcr.io/linkerd/dev:v38-rust-musl
    steps:
-      - if: matrix.arch == 'arm64'
-        run: |
-          rustup target add aarch64-unknown-linux-gnu
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            binutils-aarch64-linux-gnu \
-            g++-aarch64-linux-gnu \
-            gcc-aarch64-linux-gnu \
-            libc6-dev-arm64-cross
-          echo CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc >> "$GITHUB_ENV"
-
-      - if: matrix.arch == 'arm'
-        run: |
-          rustup target add armv7-unknown-linux-gnueabihf
-          apt-get update
-          apt-get install -y --no-install-recommends \
-            binutils-arm-linux-gnueabihf \
-            g++-arm-linux-gnueabihf \
-            gcc-arm-linux-gnueabihf \
-            libc6-dev-armhf-cross
-          echo CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc >> "$GITHUB_ENV"
-
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: just fetch
-      - name: Run just package
-        run: just profile=release package_arch=${{ matrix.arch }} package_version=${{ needs.meta.outputs.version }} package
+      - run: just arch=${{ matrix.arch }} libc=${{ matrix.libc }} rustup
+      - run: just arch=${{ matrix.arch }} libc=${{ matrix.libc }} profile=release build
+      - run: just arch=${{ matrix.arch }} libc=${{ matrix.libc }} profile=release package_version=${{ needs.meta.outputs.version }} package
      - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
        with:
          name: ${{ matrix.arch }}-artifacts
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -24,7 +24,7 @@ jobs:
  meshtls:
    timeout-minutes: 10
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    steps:
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: just fetch
@ -42,7 +42,7 @@ jobs:
  unit:
    timeout-minutes: 10
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    steps:
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: just fetch
--- a/.github/workflows/toolchain.yml
+++ b/.github/workflows/toolchain.yml
@ -14,7 +14,7 @@ permissions:
 jobs:
  devcontainer:
    runs-on: ubuntu-latest
-    container: ghcr.io/linkerd/dev:v37-rust
+    container: ghcr.io/linkerd/dev:v38-rust
    steps:
      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
      - run: |
@ -24,22 +24,6 @@ jobs:
            exit 1
          fi

-  dockerfiles:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
-      - run: |
-          VERSION="$(cat rust-toolchain)"
-          ex=0
-          while IFS= read -r file ; do
-            versions=$(sed -nE 's/^ARG RUST_VERSION=([^ ]+)/\1/p' "$file")
-            for mismatch in $(echo "$versions" | grep -vF "$VERSION" || true) ; do
-              echo "::error file=$file::$file uses incorrect rust version(s): $mismatch"
-              ex=$((ex + 1))
-            done
-          done < <(find . -name Dockerfile)
-          exit $ex
-
  workflows:
    runs-on: ubuntu-latest
    steps:
--- a/Cargo.toml
+++ b/Cargo.toml
@ -64,7 +64,7 @@ members = [
    "linkerd/transport-metrics",
    "linkerd2-proxy",
    "opencensus-proto",
-    "tools"
+    "tools",
 ]

 # Debug symbols end up chewing up several GB of disk space, so better to just
--- a/49
+++ b/49
@ -16,50 +16,39 @@
 #
 #     :; docker buildx build . --load

-ARG RUST_VERSION=1.64.0
-ARG RUST_IMAGE=rust:${RUST_VERSION}-bullseye
+ARG RUST_IMAGE=ghcr.io/linkerd/dev:v38-rust

 # Use an arbitrary ~recent edge release image to get the proxy
 # identity-initializing and linkerd-await wrappers.
-ARG RUNTIME_IMAGE=ghcr.io/linkerd/proxy:edge-22.2.1
+ARG RUNTIME_IMAGE=ghcr.io/linkerd/proxy:edge-22.12.1

 # Build the proxy, leveraging (new, experimental) cache mounting.
 #
 # See: https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/experimental.md#run---mounttypecache
-FROM $RUST_IMAGE as build
+FROM --platform=$BUILDPLATFORM $RUST_IMAGE as build

-# When set, causes the proxy to be compiled in development mode.
-ARG PROXY_UNOPTIMIZED
-
-# Controls what features are enabled in the proxy.
-ARG PROXY_FEATURES="multicore,meshtls-rustls"
-
-RUN --mount=type=cache,target=/var/lib/apt/lists \
-    --mount=type=cache,target=/var/tmp \
-  apt update && apt install -y time
-
-RUN --mount=type=cache,target=/var/lib/apt/lists \
-    --mount=type=cache,target=/var/tmp \
-  if $(echo "$PROXY_FEATURES" | grep "meshtls-boring" >/dev/null); then \
-    apt install -y cmake clang golang ; \
-  fi
+ARG PROXY_FEATURES=""
+RUN apt-get update && \
+    apt-get install -y time && \
+    if [[ "$PROXY_FEATURES" =~ .*meshtls-boring.* ]] ; then \
+      apt-get install -y golang ; \
+    fi && \
+    rm -rf /var/lib/apt/lists/*

 WORKDIR /usr/src/linkerd2-proxy
 COPY . .
-RUN --mount=type=cache,target=target \
-    --mount=type=cache,from=rust:1.64.0-bullseye,source=/usr/local/cargo,target=/usr/local/cargo \
-  mkdir -p /out && \
-  if [ -n "$PROXY_UNOPTIMIZED" ]; then \
-    (cd linkerd2-proxy && /usr/bin/time -v cargo build --locked --no-default-features --features="$PROXY_FEATURES") && \
-    mv target/debug/linkerd2-proxy /out/linkerd2-proxy ; \
-  else \
-    (cd linkerd2-proxy && /usr/bin/time -v cargo build --locked --no-default-features --features="$PROXY_FEATURES" --release) && \
-    mv target/release/linkerd2-proxy /out/linkerd2-proxy ; \
-  fi
+RUN --mount=type=cache,id=cargo,target=/usr/local/cargo/registry \
+    just fetch
+ARG TARGETARCH="amd64"
+ARG PROFILE="release"
+RUN --mount=type=cache,id=target,target=target \
+    --mount=type=cache,id=cargo,target=/usr/local/cargo/registry \
+    just arch=$TARGETARCH features=$PROXY_FEATURES profile=$PROFILE build && \
+    bin=$(just --evaluate profile="$PROFILE" _target_bin) ; \
+    mkdir -p /out && mv $bin /out/linkerd2-proxy

 ## Install the proxy binary into the base runtime image.
 FROM $RUNTIME_IMAGE as runtime
-
 WORKDIR /linkerd
 COPY --from=build /out/linkerd2-proxy /usr/lib/linkerd/linkerd2-proxy
 ENV LINKERD2_PROXY_LOG=warn,linkerd=info
--- a/89
+++ b/89
@ -12,7 +12,6 @@ export PROTOC_NO_VENDOR := "1"
 # By default we compile in development mode mode because it's faster.
 profile := if env_var_or_default("RELEASE", "") == "" { "debug" } else { "release" }
 toolchain := ""
-_cargo := "just-cargo profile=" + profile + " toolchain=" + toolchain

 features := ""

@ -25,25 +24,28 @@ docker_tag := `git rev-parse --abbrev-ref HEAD | sed 's|/|.|'` + "." + `git rev-
 docker_image := docker_repo + ":" + docker_tag

 # The architecture name to use for packages. Either 'amd64', 'arm64', or 'arm'.
-package_arch := "amd64"
+arch := "amd64"

-# If a `package_arch` is specified, then we change the default cargo `--target`
+libc := 'gnu'
+
+# If a `arch` is specified, then we change the default cargo `--target`
 # to support cross-compilation. Otherwise, we use `rustup` to find the default.
-cargo_target := if package_arch == "arm64" {
-        "aarch64-unknown-linux-gnu"
-    } else if package_arch == "arm" {
-        "armv7-unknown-linux-gnueabihf"
+_target := if arch == 'amd64' {
+        "x86_64-unknown-linux-" + libc
+    } else if arch == "arm64" {
+        "aarch64-unknown-linux-" + libc
+    } else if arch == "arm" {
+        "armv7-unknown-linux-" + libc + "eabihf"
    } else {
-        `rustup show | sed -n 's/^Default host: \(.*\)/\1/p'`
+        error("unsupported arch=" + arch)
    }

-# Support cross-compilation when `package_arch` changes.
-strip := if package_arch == "arm64" { "aarch64-linux-gnu-strip" } else if package_arch == "arm" { "arm-linux-gnueabihf-strip" } else { "strip" }
+_cargo := 'just-cargo profile=' + profile + ' target=' + _target + ' toolchain=' + toolchain

-target_dir := join("target", cargo_target, profile)
-target_bin := join(target_dir, "linkerd2-proxy")
-package_name := "linkerd2-proxy-" + package_version + "-" + package_arch
-package_dir := join("target/package", package_name)
+_target_dir := "target" / _target / profile
+_target_bin := _target_dir / "linkerd2-proxy"
+_package_name := "linkerd2-proxy-" + package_version + "-" + arch + if libc == 'musl' { '-static' } else { '' }
+_package_dir := "target/package" / _package_name
 shasum := "shasum -a 256"

 _features := if features == "all" {
@ -56,6 +58,9 @@ _features := if features == "all" {
 # Recipes
 #

+rustup:
+    @{{ _cargo }} _target-installed
+
 # Run all lints
 lint: sh-lint md-lint clippy doc action-lint action-dev-check

@ -106,27 +111,40 @@ test-dir dir *flags:
    cd {{ dir }} && {{ _cargo }} test --frozen {{ _features }} {{ flags }}

 # Build the proxy
-build:
-    @{{ _cargo }} build --frozen --package=linkerd2-proxy --target={{ cargo_target }} {{ _features }}
+build: && checksec _strip
+    @rm -f {{ _target_bin }} {{ _target_bin }}.dbg
+    @{{ _cargo }} build --frozen --package=linkerd2-proxy {{ _features }}

-_package_bin := package_dir / "bin" / "linkerd2-proxy"
+_strip:
+    {{ _objcopy }} --only-keep-debug {{ _target_bin }} {{ _target_bin }}.dbg
+    {{ _objcopy }} --strip-unneeded {{ _target_bin }}
+    {{ _objcopy }} --add-gnu-debuglink={{ _target_bin }}.dbg {{ _target_bin }}
+
+_package_bin := _package_dir / "bin" / "linkerd2-proxy"
+
+# XXX {aarch64,arm}-musl builds do not enable PIE, so we use target-specific
+# files to document those differences.
+_expected_checksec := '.checksec' / arch + '-' + libc + '.json'
+
+# Check the security properties of the proxy binary.
+checksec:
+    checksec --output=json --file='{{ _target_bin }}' \
+        | jq '.' | tee /dev/stderr \
+        | jq -S '.[] | del(."fortify_source") | del(."fortify-able") | del(.fortified) | del(.symbols)' \
+        | diff -u {{ _expected_checksec }} - >&2
+
+_objcopy := 'llvm-objcopy-' + `just-cargo --evaluate _llvm-version`

 # Build a package (i.e. for a release)
 package: build
-    mkdir -p {{ package_dir }}/bin
-    cp LICENSE {{ package_dir }}/
-    cp {{ target_dir }}/linkerd2-proxy {{ _package_bin }}
-    {{ strip }} {{ _package_bin }}
-    checksec --output=json --file='{{ _package_bin }}' \
-        | jq '.["{{ _package_bin }}"] | del(."fortify-able") | del(.fortified)' \
-        > target/package/{{ package_name }}-checksec.json
-    jq -S '.'  target/package/{{ package_name }}-checksec.json \
-        | diff -u .checksec-expected.json - >&2
-    cd target/package \
-        && (tar -czvf {{ package_name }}.tar.gz {{ package_name }} >/dev/null) \
-        && ({{ shasum }} {{ package_name }}.tar.gz > {{ package_name }}.txt)
-    @rm -rf {{ package_dir }}
-    @du -h target/package/{{ package_name }}*
+    @mkdir -p {{ _package_dir }}/bin
+    cp LICENSE {{ _package_dir }}/
+    cp {{ _target_bin }} {{ _target_bin }}.dbg {{ _package_dir }}/
+    tar -czvf target/package/{{ _package_name }}.tar.gz  -C target/package {{ _package_name }} >/dev/null
+    cd target/package && ({{ shasum }} {{ _package_name }}.tar.gz | tee {{ _package_name }}.txt)
+    @rm -rf {{ _package_dir }}
+    @du -h target/package/{{ _package_name }}.tar.gz
+    @tar tzvf target/package/{{ _package_name }}.tar.gz

 # Build all of the fuzzers (SLOW).
 fuzzers:
@ -141,18 +159,19 @@ fuzzers:
        echo "cd $dir && {{ _cargo }} fuzz build"
        (
            cd $dir
-            @{{ _cargo }} fuzz build --target={{ cargo_target }} \
+            @{{ _cargo }} fuzz build \
                {{ if profile == "release" { "--release" } else { "" } }}
        )
    done

 # Build a docker image (FOR TESTING ONLY)
-docker mode='load':
+docker *args='--output=type=docker':
    docker buildx build . \
+        --pull \
        --tag={{ docker_image }} \
-        {{ if profile != 'release' { "--build-arg PROXY_UNOPTIMIZED=1" } else { "" } }} \
+        --build-arg PROFILE='{{ profile }}' \
        {{ if features != "" { "--build-arg PROXY_FEATURES=" + features } else { "" } }} \
-        {{ if mode == 'push' { "--push" } else { "--load" } }}
+        {{ args }}

 # Lints all shell scripts in the repo.
 sh-lint: