Update gRPC & protobuf dependencies (#8218)

This change updates the repo to use `linkerd2-proxy-api` v0.4.0 and
updates `bin/protoc` to use v3.20 to match the configuration in the
other repo.

The policy-controller builds are updated to use our `bin/protoc` wrapper
so that all builds go through the same toolchain (and to avoid compiling
protoc on each build).

Signed-off-by: Oliver Gould <ver@buoyant.io>

.devcontainer/Dockerfile

@@ -13,6 +13,8 @@ RUN for p in \
     github.com/go-delve/delve/cmd/dlv@latest \
     github.com/golangci/golangci-lint/cmd/golangci-lint@${GOLANGCI_LINT_VERSION} \
     golang.org/x/tools/gopls@latest \
+    google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 \
+    google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 \
     ; do go install "$p" ; done
 
 FROM docker.io/golang:${GO_VERSION}-bullseye as cargo-deny

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
     "name": "linkerd2",
-    "image": "ghcr.io/linkerd/dev:v10",
+    "image": "ghcr.io/linkerd/dev:v11",
     // "dockerFile": "./Dockerfile",
     // "context": "..",
     "extensions": [

.dockerignore

@@ -7,8 +7,9 @@
 **/node_modules
 bin
 !bin/fetch-proxy
-!bin/scurl
 !bin/install-deps
+!bin/protoc
+!bin/scurl
 !bin/web
 **/Dockerfile*
 Dockerfile*

.github/workflows/codecov.yml

@@ -17,7 +17,7 @@ jobs:
       image: golang:1.17.3
     steps:
       - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
-      - run: go get gotest.tools/gotestsum@v0.4.2
+      - run: go install gotest.tools/gotestsum@v0.4.2
       - run: gotestsum -- -cover -coverprofile=coverage.out -v -mod=readonly ./...
       - uses: codecov/codecov-action@e3c560433a6cc60aec8812599b7844a7b4fa0d71
         with:

.github/workflows/policy_controller.yml

@@ -24,6 +24,8 @@ env:
   PROXY_INIT_VERSION: v1.5.3
   RUST_BACKTRACE: short
   RUSTUP_MAX_RETRIES: 10
+  PROTOC_NO_VENDOR: 1
+
 
 jobs:
   fmt:
@@ -64,7 +66,8 @@ jobs:
           bin/scurl -o /usr/local/bin/cargo-action-fmt "https://github.com/olix0r/cargo-action-fmt/releases/download/release%2F${CARGO_ACTION_FMT_VERSION}/cargo-action-fmt-x86_64-unknown-linux-gnu"
           chmod 755 /usr/local/bin/cargo-action-fmt
       - run: cargo fetch --locked
-      - run: cargo clippy --frozen --all --no-deps --message-format=json | cargo-action-fmt
+      - name: Run cargo clippy
+        run: PROTOC=$PWD/bin/protoc cargo clippy --frozen --all --no-deps --message-format=json | cargo-action-fmt
 
   check:
     timeout-minutes: 20
@@ -79,6 +82,7 @@ jobs:
       - run: cargo fetch --locked
       # Check each crate independently to ensure its Cargo.toml is sufficient.
       - run: |
+          export PROTOC=$PWD/bin/protoc
           for toml in $(find . -mindepth 2 -name Cargo.toml | sort -r)
           do
             d=$(dirname "$toml")
@@ -95,8 +99,10 @@ jobs:
     steps:
       - uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
       - run: cargo fetch --locked
-      - run: cargo test --workspace --exclude=linkerd-policy-test --frozen --no-run
+      - name: Build tests
-      - run: cargo test --workspace --exclude=linkerd-policy-test --frozen
+        run: PROTOC=$PWD/bin/protoc cargo test --workspace --exclude=linkerd-policy-test --frozen --no-run
+      - name: Run cargo test
+        run: PROTOC=$PWD/bin/protoc cargo test --workspace --exclude=linkerd-policy-test --frozen
 
   rust-toolchain:
     name: rust toolchain
@@ -189,7 +195,8 @@ jobs:
       # chance to settle down before loading images, etc. It also ensures that the tests compile
       # before doing extra work.
       - run: cargo fetch --locked
-      - run: cargo test -p linkerd-policy-test --frozen --no-run | cargo-action-fmt
+      - name: Build tests
+        run: PROTOC=$PWD/bin/protoc cargo test -p linkerd-policy-test --frozen --no-run | cargo-action-fmt
 
       - name: Download image archives
         uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741
@@ -235,4 +242,5 @@ jobs:
 
       # Run the tests. We disable parallelism to avoid spurious timeouts caused
       # by resource contention.
-      - run: cargo test -p linkerd-policy-test --frozen -- --test-threads=1
+      - name: Run cargo test
+        run: PROTOC=$PWD/bin/protoc cargo test -p linkerd-policy-test --frozen -- --test-threads=1

.github/workflows/unit_tests.yml

@@ -15,9 +15,12 @@ jobs:
     steps:
     - name: Checkout code
       uses: actions/checkout@a12a3943b4bdde767164f792f33f40b04645d846
+    - run: go mod download
+    - run: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2
+    - run: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28
     - name: Go unit tests
       run: |
-        go get gotest.tools/gotestsum@v0.4.2
+        go install gotest.tools/gotestsum@v0.4.2
         # TODO: validate bin/protoc-go.sh does not dirty the repo
         LINKERD_TEST_PRETTY_DIFF=1 gotestsum -- -race -v -mod=readonly ./...
 

Cargo.lock

@@ -13,15 +13,6 @@ dependencies = [
  "version_check",
 ]
 
-[[package]]
-name = "aho-corasick"
-version = "0.7.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "ansi_term"
 version = "0.12.1"
@@ -75,6 +66,49 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
+[[package]]
+name = "axum"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47594e438a243791dba58124b6669561f5baa14cb12046641d8008bf035e5a25"
+dependencies = [
+ "async-trait",
+ "axum-core",
+ "bitflags",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde",
+ "sync_wrapper",
+ "tokio",
+ "tower",
+ "tower-http",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a671c9ae99531afdd5d3ee8340b8da547779430689947144c140fc74a740244"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "mime",
+]
+
 [[package]]
 name = "backoff"
 version = "0.4.0"
@@ -170,13 +204,22 @@ version = "3.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1"
 dependencies = [
- "heck 0.4.0",
+ "heck",
  "proc-macro-error",
  "proc-macro2",
  "quote",
  "syn",
 ]
 
+[[package]]
+name = "cmake"
+version = "0.1.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "core-foundation"
 version = "0.9.3"
@@ -497,15 +540,6 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 
-[[package]]
-name = "heck"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
-dependencies = [
- "unicode-segmentation",
-]
-
 [[package]]
 name = "heck"
 version = "0.4.0"
@@ -1026,9 +1060,9 @@ dependencies = [
 
 [[package]]
 name = "linkerd2-proxy-api"
-version = "0.3.1"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6aaf91178c272abeaac52b2472351e73affa723bfdd0c15e147e2f975f2fbe5"
+checksum = "a76fcbfd8f3f305638c81aa5e42f289db74cb71cec785be6a7fb55bf68791187"
 dependencies = [
  "ipnet",
  "prost",
@@ -1078,12 +1112,24 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
 
+[[package]]
+name = "matchit"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
+
 [[package]]
 name = "memchr"
 version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
 
+[[package]]
+name = "mime"
+version = "0.3.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
+
 [[package]]
 name = "mio"
 version = "0.8.2"
@@ -1318,6 +1364,16 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
 
+[[package]]
+name = "prettyplease"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b83ec2d0af5c5c556257ff52c9f98934e243b9fd39604bfb2a9b75ec2e97f18"
+dependencies = [
+ "proc-macro2",
+ "syn",
+]
+
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -1353,9 +1409,9 @@ dependencies = [
 
 [[package]]
 name = "prost"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
+checksum = "1bd5316aa8f5c82add416dfbc25116b84b748a21153f512917e8143640a71bbd"
 dependencies = [
  "bytes",
  "prost-derive",
@@ -1363,12 +1419,14 @@ dependencies = [
 
 [[package]]
 name = "prost-build"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
+checksum = "328f9f29b82409216decb172d81e936415d21245befa79cd34c3f29d87d1c50b"
 dependencies = [
  "bytes",
- "heck 0.3.3",
+ "cfg-if",
+ "cmake",
+ "heck",
  "itertools",
  "lazy_static",
  "log",
@@ -1383,9 +1441,9 @@ dependencies = [
 
 [[package]]
 name = "prost-derive"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
+checksum = "df35198f0777b75e9ff669737c6da5136b59dba33cf5a010a6d1cc4d56defc6f"
 dependencies = [
  "anyhow",
  "itertools",
@@ -1396,9 +1454,9 @@ dependencies = [
 
 [[package]]
 name = "prost-types"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
+checksum = "926681c118ae6e512a3ccefd4abbe5521a14f4cc1e207356d4d00c0b7f2006fd"
 dependencies = [
  "bytes",
  "prost",
@@ -1469,8 +1527,6 @@ version = "1.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
 dependencies = [
- "aho-corasick",
- "memchr",
  "regex-syntax",
 ]
 
@@ -1783,6 +1839,12 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "sync_wrapper"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"
+
 [[package]]
 name = "tempfile"
 version = "3.3.0"
@@ -1977,12 +2039,13 @@ dependencies = [
 
 [[package]]
 name = "tonic"
-version = "0.6.2"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a"
+checksum = "30fb54bf1e446f44d870d260d99957e7d11fb9d0a0f5bd1a662ad1411cc103f9"
 dependencies = [
  "async-stream",
  "async-trait",
+ "axum",
  "base64",
  "bytes",
  "futures-core",
@@ -1998,7 +2061,7 @@ dependencies = [
  "prost-derive",
  "tokio",
  "tokio-stream",
- "tokio-util 0.6.9",
+ "tokio-util 0.7.0",
  "tower",
  "tower-layer",
  "tower-service",
@@ -2008,10 +2071,11 @@ dependencies = [
 
 [[package]]
 name = "tonic-build"
-version = "0.6.2"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757"
+checksum = "4d17087af5c80e5d5fc8ba9878e60258065a0a757e35efe7a05b7904bece1943"
 dependencies = [
+ "prettyplease",
  "proc-macro2",
  "prost-build",
  "quote",
@@ -2053,6 +2117,7 @@ dependencies = [
  "http-body",
  "http-range-header",
  "pin-project-lite",
+ "tower",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -2211,12 +2276,6 @@ dependencies = [
  "tinyvec",
 ]
 
-[[package]]
-name = "unicode-segmentation"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
-
 [[package]]
 name = "unicode-xid"
 version = "0.2.2"

bin/protoc

@@ -11,7 +11,7 @@ arch=$(uname -m)
 
 bindir=$( cd "${0%/*}" && pwd )
 targetbin=$( cd "$bindir"/.. && pwd )/target/bin
-protocversion=3.15.8
+protocversion=3.20.0
 protocbin=$targetbin/protoc-$protocversion
 protocurl=https://github.com/google/protobuf/releases/download/v$protocversion/protoc-$protocversion-$os-$arch.zip
 

bin/protoc-go.sh

@@ -4,8 +4,8 @@ set -eu
 
 bindir=$( cd "${0%/*}" && pwd )
 
-go install -mod=readonly github.com/golang/protobuf/protoc-gen-go
+go install -mod=readonly google.golang.org/protobuf/cmd/protoc-gen-go@v1.28
-go install -mod=readonly google.golang.org/grpc/cmd/protoc-gen-go-grpc
+go install -mod=readonly google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2
 
 rm -rf controller/gen/common controller/gen/config viz/metrics-api/gen viz/tap/gen
 mkdir -p controller/gen/common/net viz/metrics-api/gen/viz viz/tap/gen/tap

controller/gen/common/net/net.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.28.0
-// 	protoc        v3.15.8
+// 	protoc        v3.20.0
 // source: common/net.proto
 
 package net

deny.toml

@@ -44,9 +44,6 @@ wildcards = "allow"
 highlight = "all"
 deny = []
 skip = [
-    # Waiting on a prost-build release that includes
-    # https://github.com/tokio-rs/prost/pull/583
-    { name = "heck", version = "0.3" },
     # Waiting on h2, tonic, etc
     { name = "tokio-util", version = "0.6" },
     # Waiting on hyper-rustls, etc

policy-controller/amd64.dockerfile

@@ -5,6 +5,10 @@ ARG RUNTIME_IMAGE=gcr.io/distroless/cc
 FROM $RUST_IMAGE as build
 ARG TARGETARCH
 WORKDIR /build
+COPY bin/scurl bin/scurl
+COPY bin/protoc bin/protoc
+ENV PROTOC_NO_VENDOR=1
+ENV PROTOC=/build/bin/protoc
 COPY Cargo.toml Cargo.lock .
 COPY policy-controller policy-controller
 RUN cargo new policy-test --lib

policy-controller/arm.dockerfile

@@ -8,6 +8,10 @@ RUN apt-get update && \
     rustup target add armv7-unknown-linux-gnueabihf
 ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc
 WORKDIR /build
+COPY bin/scurl bin/scurl
+COPY bin/protoc bin/protoc
+ENV PROTOC_NO_VENDOR=1
+ENV PROTOC=/build/bin/protoc
 COPY Cargo.toml Cargo.lock .
 COPY policy-controller policy-controller
 RUN cargo new policy-test --lib

policy-controller/arm64.dockerfile

@@ -8,6 +8,10 @@ RUN apt-get update && \
     rustup target add aarch64-unknown-linux-gnu
 ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
 WORKDIR /build
+COPY bin/scurl bin/scurl
+COPY bin/protoc bin/protoc
+ENV PROTOC_NO_VENDOR=1
+ENV PROTOC=/build/bin/protoc
 COPY Cargo.toml Cargo.lock .
 COPY policy-controller policy-controller
 RUN cargo new policy-test --lib

policy-controller/grpc/Cargo.toml

@@ -5,17 +5,14 @@ edition = "2021"
 license = "Apache-2.0"
 publish = false
 
-[features]
-rustfmt = ["linkerd2-proxy-api/rustfmt"]
-
 [dependencies]
 async-stream = "0.3"
 async-trait = "0.1"
 drain = "0.1"
 futures = { version = "0.3", default-features = false }
-linkerd2-proxy-api = { version = "0.3", features = ["inbound", "server"] }
+linkerd2-proxy-api = { version = "0.4", features = ["inbound", "server"] }
 linkerd-policy-controller-core = { path = "../core" }
 maplit = "1"
 tokio = { version = "1", features = ["macros"] }
-tonic = { version = "0.6", default-features = false, features = ["transport"] }
+tonic = { version = "0.7", default-features = false, features = ["transport"] }
 tracing = "0.1"

policy-test/Cargo.toml

@@ -14,13 +14,13 @@ ipnet = "2"
 k8s-openapi = { version = "0.14", features = ["v1_20"] }
 linkerd-policy-controller-core = { path = "../policy-controller/core" }
 linkerd-policy-controller-k8s-api = { path = "../policy-controller/k8s/api" }
-linkerd2-proxy-api = { version = "0.3", features = ["inbound", "client"] }
+linkerd2-proxy-api = { version = "0.4", features = ["inbound", "client"] }
 maplit = "1"
 rand = "0.8"
 serde = "1"
 serde_json = "1"
 schemars = "0.8"
-tonic = { version = "0.6" }
+tonic = { version = "0.7" }
 tokio = { version = "1", features = ["macros", "rt"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }

viz/metrics-api/gen/viz/viz.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.28.0
-// 	protoc        v3.15.8
+// 	protoc        v3.20.0
 // source: viz.proto
 
 package viz

viz/metrics-api/gen/viz/viz_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.2.0
-// - protoc             v3.15.8
+// - protoc             v3.20.0
 // source: viz.proto
 
 package viz

viz/tap/gen/tap/viz_tap.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.28.0
-// 	protoc        v3.15.8
+// 	protoc        v3.20.0
 // source: viz_tap.proto
 
 package tap

viz/tap/gen/tap/viz_tap_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.2.0
-// - protoc             v3.15.8
+// - protoc             v3.20.0
 // source: viz_tap.proto
 
 package tap