build: Update fetch-proxy to support alternate repos (#12135)

* build: Update fetch-proxy to support alternate repos

In some build and test scenarios, it's desirable to fetch alternate proxy
releases during the build process.

This change updates the proxy container image build tooling to support the
LINKERD2_PROXY_REPO and LINKERD2_PROXY_GITHUB_TOKEN environment variables. These
may be set to the desired repository and a GitHub personal access token,
respectively.

When these are unset, the default behavior is unchanged.

.github/workflows/integration.yml

@@ -35,6 +35,7 @@ env:
   RUST_BACKTRACE: short
   RUSTUP_MAX_RETRIES: 10
   YQ_VERSION: v4.25.1
+  LINKERD2_PROXY_REPO: ${{ vars.LINKERD2_PROXY_REPO }}
 
 jobs:
   cleanup:
@@ -107,6 +108,8 @@ jobs:
           docker-target: linux-amd64
           component: ${{ matrix.component }}
           tag: ${{ needs.tag.outputs.tag }}
+        env:
+          LINKERD2_PROXY_GITHUB_TOKEN: ${{ secrets.LINKERD2_PROXY_GITHUB_TOKEN }}
       - name: Run docker save
         run: |
           mkdir -p /home/runner/archives

.github/workflows/release.yml

@@ -13,6 +13,7 @@ env:
   GH_ANNOTATION: true
   DOCKER_REGISTRY: ghcr.io/linkerd
   K3D_VERSION: v5.4.4
+  LINKERD2_PROXY_REPO: ${{ vars.LINKERD2_PROXY_REPO }}
 
 jobs:
   # TODO(ver) We should stop relying so heavily on the environment,
@@ -61,6 +62,8 @@ jobs:
         docker-ghcr-pat: ${{ secrets.DOCKER_GHCR_PAT }}
         component: ${{ matrix.component }}
         tag: ${{ needs.tag.outputs.tag }}
+      env:
+        LINKERD2_PROXY_GITHUB_TOKEN: ${{ secrets.LINKERD2_PROXY_GITHUB_TOKEN }}
     - uses: sigstore/cosign-installer@v3
     - run: cosign sign '${{ steps.build.outputs.digest }}'
       env:

Dockerfile-proxy

@@ -10,14 +10,22 @@ RUN go mod download
 ARG TARGETARCH
 RUN ./bin/install-deps $TARGETARCH
 
-FROM --platform=$BUILDPLATFORM curlimages/curl:7.86.0 as fetch
+FROM --platform=$BUILDPLATFORM debian:bookworm-slim as fetch
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq && \
+    rm -rf /var/lib/apt/lists/*
 WORKDIR /build
 COPY bin/fetch-proxy bin/fetch-proxy
 COPY bin/scurl bin/scurl
-COPY .proxy-version proxy-version
 ARG TARGETARCH
-RUN (proxy=$(bin/fetch-proxy $(cat proxy-version) $TARGETARCH) && \
-    mv "$proxy" linkerd2-proxy)
+ARG LINKERD2_PROXY_REPO="linkerd/linkerd2-proxy"
+ARG LINKERD2_PROXY_RELEASE_PREFIX="release/"
+ARG LINKERD2_PROXY_VERSION=""
+RUN --mount=type=secret,id=github \
+    export GITHUB_TOKEN_FILE=/run/secrets/github; \
+    proxy=$(bin/fetch-proxy "$LINKERD2_PROXY_VERSION" "$TARGETARCH"); \
+    mv "$proxy" linkerd2-proxy
+RUN echo "$LINKERD2_PROXY_VERSION" > proxy-version
 ARG LINKERD_AWAIT_VERSION=v0.2.6
 RUN bin/scurl -o linkerd-await https://github.com/linkerd/linkerd-await/releases/download/release%2F${LINKERD_AWAIT_VERSION}/linkerd-await-${LINKERD_AWAIT_VERSION}-${TARGETARCH} && chmod +x linkerd-await
 ARG LINKERD_VALIDATOR_VERSION=v0.1.2

bin/docker-build-proxy

@@ -28,5 +28,9 @@ get_extra_options() {
 # We want wordsplit for the extra options here:
 # shellcheck disable=SC2046
 docker_build proxy "${TAG:-$(head_root_tag)}" "$dockerfile" \
-  --build-arg "LINKERD_VERSION=${TAG:-$(head_root_tag)}" \
+  --build-arg LINKERD_VERSION="${TAG:-$(head_root_tag)}" \
+  --build-arg LINKERD2_PROXY_REPO="${LINKERD2_PROXY_REPO:-linkerd/linkerd2-proxy}" \
+  --build-arg LINKERD2_PROXY_RELEASE_PREFIX="${LINKERD2_PROXY_RELEASE_PREFIX:-release/}" \
+  --build-arg LINKERD2_PROXY_VERSION="${LINKERD2_PROXY_VERSION:-$(cat .proxy-version)}" \
+  --secret id=github,env=LINKERD2_PROXY_GITHUB_TOKEN \
   $(get_extra_options)

bin/fetch-proxy

@@ -11,21 +11,49 @@ bindir=$( cd "${0%/*}" && pwd )
 rootdir=$( cd "$bindir"/.. && pwd )
 builddir="$rootdir/target/proxy"
 
-version=${1:-latest}
-if [ "$version" = latest ]; then
-  version=$("$bindir"/scurl https://api.github.com/repos/linkerd/linkerd2-proxy/releases/latest |jq -r .tag_name | sed 's,^release/,,')
+proxy_repo="${LINKERD2_PROXY_REPO:-}"
+if [ -z "$proxy_repo" ]; then
+  proxy_repo=linkerd/linkerd2-proxy
 fi
 
-assetbase="https://github.com/linkerd/linkerd2-proxy/releases/download/release%2F${version}"
-arch=${2:-amd64}
-pkgname="linkerd2-proxy-${version}-${arch}"
-pkgfile="${pkgname}.tar.gz"
-shafile="${pkgname}.txt"
+releases_url=https://api.github.com/repos/"$proxy_repo"/releases
+release_prefix="${LINKERD2_PROXY_RELEASE_PREFIX:-release/}"
+
+github_token="${GITHUB_TOKEN:-}"
+if [ -z "$github_token" ] && [ -n "${GITHUB_TOKEN_FILE:-}" ] && [ -f "$GITHUB_TOKEN_FILE" ]; then
+  github_token=$(cat "$GITHUB_TOKEN_FILE")
+fi
+
+ghcurl() {
+  if [ -n "${github_token:-}" ]; then
+    "$bindir"/scurl -H "Authorization: Bearer ${github_token:-}" "$@"
+  else
+    "$bindir"/scurl "$@"
+  fi
+}
 
 mkdir -p "$builddir"
 cd "$builddir"
-"$bindir"/scurl -O "$assetbase/$pkgfile"
-"$bindir"/scurl -O "$assetbase/$shafile"
+
+version=${1:-latest}
+arch=${2:-amd64}
+if [ "$version" = latest ]; then
+  ghcurl "$releases_url"/latest > release.json
+  version=$(jq -r .tag_name release.json | sed 's,^'"${release_prefix}"',,')
+else
+  tag="${release_prefix}${version}"
+  ghcurl "$releases_url"/tags/"$(printf "$tag" | jq -sRr @uri)" > release.json
+fi
+
+pkgname="linkerd2-proxy-${version}-${arch}"
+
+pkgfile="${pkgname}.tar.gz"
+pkgurl=$(jq -r '.assets[] | select(.name == "'"$pkgfile"'") | .url' release.json)
+ghcurl -H 'Accept: application/octet-stream' -o "$pkgfile" "$pkgurl"
+
+shafile="${pkgname}.txt"
+shaurl=$(jq -r '.assets[] | select(.name == "'"$shafile"'") | .url' release.json)
+ghcurl -H 'Accept: application/octet-stream' -o "$shafile" "$shaurl"
 
 tar -zxvf "$pkgfile" >&2
 expected=$(awk '{print $1}' "$shafile")

bin/git-commit-proxy-version

@@ -33,11 +33,15 @@ new_proxy_rev="release/$new_proxy_version"
 # Checkout the linkerd2-proxy repo to resolve the new proxy version to a SHA
 # and obtain the commit log since the prior version.
 tmp=$(mktemp -d -t l2-proxy.XXX)
-git clone --depth=500 https://github.com/linkerd/linkerd2-proxy "$tmp"
+repo="${LINKERD2_PROXY_REPO:-linkerd/linkerd2-proxy}"
+git clone --depth=500 https://github.com/"$repo" "$tmp"
 cd "$tmp"
 
 if ! git rev-parse --verify --quiet "${old_proxy_rev}" ; then
-    echo "The old proxy-version ${old_proxy_version} does not exist in the last 100 proxy commits." >&2
+    git pull --tags
+fi
+if ! git rev-parse --verify --quiet "${old_proxy_rev}" ; then
+    echo "The old proxy-version ${old_proxy_version} not found." >&2
     exit 2
 fi