scripts: regenerate pbs with caching deps to a fixed tmp folder (#7409)

scripts/install-protoc.sh

@@ -16,19 +16,15 @@
 #
 # install-protoc.sh
 #
-# This script installs the Protocol Buffers compiler (protoc) on the local
+# This script installs the Protocol Buffers compiler (protoc) to the specified
-# machine. It is used to generate code from .proto files for gRPC communication.
+# directory.  It is used to generate code from .proto files for gRPC
-# The script downloads the protoc binary from the official GitHub repository and
+# communication. The script downloads the protoc binary from the official GitHub
-# installs it in the system. 
+# repository and installs it in the system.
 #
-# Usage: ./install-protoc.sh [INSTALL_PATH]
+# Usage: ./install-protoc.sh INSTALL_PATH
 #
 # Arguments:
-#   INSTALL_PATH: The path where the protoc binary will be installed (optional).
+#   INSTALL_PATH: The path where the protoc binary will be installed.
-#                 If not provided, the script will install the binary in the the
-#                 directory named by the GOBIN environment variable, which
-#                 defaults to $GOPATH/bin or $HOME/go/bin if the GOPATH
-#                 environment variable is not set. 
 #
 # Note: This script requires internet connectivity to download the protoc binary.
 
@@ -39,53 +35,60 @@ source "$(dirname $0)/common.sh"
 # The version of protoc that will be installed.
 PROTOC_VERSION="27.1"
 
-INSTALL_PATH="${1:-${GOBIN:-${GOPATH:-$HOME/go}}}"
-
-# downloads the pre-built binaries for Linux to $INSTALL_PATH.
-# Usage:
-#   download_binary ARCH OS
-# Arguments:
-#   ARCH:  The architecture of the system.  Accepted values: [x86_64, aarch_64]
-#   OS:    The operating system of the system.  Accepted values: [osx, linux]
-download_binary() {  
-  DOWNLOAD_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-$2-$1.zip"
-  # -L follows redirects.
-  # -O writes output to a file.
-  curl -LO "${DOWNLOAD_URL}"
-  unzip "protoc-${PROTOC_VERSION}-$2-$1.zip" -d "${INSTALL_PATH}"
-  rm "protoc-${PROTOC_VERSION}-$2-$1.zip"
-  rm "${INSTALL_PATH}/readme.txt"
-}
-
 main() {
-  # Check if protoc is already available.
+  if [[ "$#" -ne 1 ]]; then
-  if command -v protoc &> /dev/null; then
+    die "Usage: $0 INSTALL_PATH"
-    if INSTALL_VERSION=$(protoc --version | cut -d' ' -f2 2>/dev/null); then
-      if [ "$INSTALL_VERSION" = "$PROTOC_VERSION" ]; then
-        echo "protoc version $PROTOC_VERSION is already installed."
-        return
-      else
-        die "Existing protoc version ($INSTALL_VERSION) differs. Kindly make sure you have $PROTOC_VERSION installed."
-      fi
-    fi
   fi
 
-  # Detect the architecture
+  INSTALL_PATH="${1}"
+
+  if [[ ! -d "${INSTALL_PATH}" ]]; then
+    die "INSTALL_PATH (${INSTALL_PATH}) does not exist."
+  fi
+
+  echo "Installing protoc version $PROTOC_VERSION to ${INSTALL_PATH}..."
+
+  # Detect the hardware platform.
   case "$(uname -m)" in
     "x86_64")   ARCH="x86_64";;
     "aarch64")  ARCH="aarch_64";;
     "arm64")    ARCH="aarch_64";;
-  *) die "Unsupported architecture. Please consider manual installation from "\
+    *)          die "Install unsuccessful. Hardware platform not supported by installer: $1";;
-          "https://github.com/protocolbuffers/protobuf/releases/ and add to PATH."
   esac
 
-  # Detect the Operating System
+  # Detect the Operating System.
   case "$(uname -s)" in
-    "Darwin") download_binary $ARCH "osx";;
+    "Darwin") OS="osx";;
-    "Linux") download_binary $ARCH "linux";;
+    "Linux")  OS="linux";;
-  *) die "Unsupported OS. Please consider manual installation from "\
+    *)        die "Install unsuccessful. OS not supported by installer: $2";;
-          "https://github.com/protocolbuffers/protobuf/releases/ and add to PATH" ;;
   esac
+
+  # Check if the protoc binary with the right version is already installed.
+  if [[ -f "${INSTALL_PATH}/bin/protoc" ]]; then
+    if [[ "$("${INSTALL_PATH}/bin/protoc" --version)" == "libprotoc ${PROTOC_VERSION}" ]]; then
+      echo "protoc version ${PROTOC_VERSION} is already installed in ${INSTALL_PATH}"
+      return
+    fi
+  fi
+
+  DOWNLOAD_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${OS}-${ARCH}.zip"
+
+  # -L follows redirects.
+  # -O writes output to a file.
+  curl -LO "${DOWNLOAD_URL}"
+
+  # Unzip the downloaded file and except readme.txt.
+  # The file structure should look like:
+  # INSTALL_PATH
+  # ├── bin
+  # │   └── protoc
+  # └── include
+  #     └── other files...
+  unzip "protoc-${PROTOC_VERSION}-${OS}-${ARCH}.zip" -d "${INSTALL_PATH}" -x "readme.txt"
+  rm "protoc-${PROTOC_VERSION}-${OS}-${ARCH}.zip"
+
+  # Make the protoc binary executable. ¯\_(ツ)_/¯  crazy, right?
+  chmod +x "${INSTALL_PATH}/bin/protoc"
 }
 
 main "$@"

scripts/regenerate.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+#
 # Copyright 2020 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,39 +16,43 @@
 
 set -eu -o pipefail
 
-WORKDIR=$(mktemp -d)
+WORKDIR="/tmp/grpc-go-tools"
+mkdir -p "${WORKDIR}"
 
-function finish {
+"$(dirname "${0}")"/install-protoc.sh ${WORKDIR}
-  rm -rf "$WORKDIR"
-}
-trap finish EXIT
-
-./scripts/install-protoc.sh "${WORKDIR}"
 
 export GOBIN="${WORKDIR}"/bin
 export PATH="${GOBIN}:${PATH}"
 mkdir -p "${GOBIN}"
 
-echo "remove existing generated files"
+echo "removing existing generated files..."
 # grpc_testing_not_regenerate/*.pb.go is not re-generated,
 # see grpc_testing_not_regenerate/README.md for details.
-rm -f $(find . -name '*.pb.go' | grep -v 'grpc_testing_not_regenerate')
+find . -name '*.pb.go' | grep -v 'grpc_testing_not_regenerate' | xargs rm -f || true
 
-echo "go install google.golang.org/protobuf/cmd/protoc-gen-go"
+echo "Executing: go install google.golang.org/protobuf/cmd/protoc-gen-go..."
 (cd test/tools && go install google.golang.org/protobuf/cmd/protoc-gen-go)
 
-echo "go install cmd/protoc-gen-go-grpc"
+echo "Executing: go install cmd/protoc-gen-go-grpc..."
 (cd cmd/protoc-gen-go-grpc && go install .)
 
-echo "git clone https://github.com/grpc/grpc-proto"
+echo "Pulling protos from https://github.com/grpc/grpc-proto..."
+if [ -d "${WORKDIR}/grpc-proto" ]; then
+  (cd "${WORKDIR}/grpc-proto" && git pull)
+else
   git clone --quiet https://github.com/grpc/grpc-proto "${WORKDIR}/grpc-proto"
+fi
 
-echo "git clone https://github.com/protocolbuffers/protobuf"
+echo "Pulling protos from https://github.com/protocolbuffers/protobuf..."
+if [ -d "${WORKDIR}/protobuf" ]; then
+  (cd "${WORKDIR}/protobuf" && git pull)
+else
   git clone --quiet https://github.com/protocolbuffers/protobuf "${WORKDIR}/protobuf"
+fi
 
 # Pull in code.proto as a proto dependency
 mkdir -p "${WORKDIR}/googleapis/google/rpc"
-echo "curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto"
+echo "Pulling code.proto from https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto..."
 curl --silent https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto > "${WORKDIR}/googleapis/google/rpc/code.proto"
 
 
@@ -72,8 +77,8 @@ SOURCES=(
   "${WORKDIR}/grpc-proto/grpc/gcp/transport_security_common.proto"
   "${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto"
   "${WORKDIR}/grpc-proto/grpc/lookup/v1/rls_config.proto"
-  "${WORKDIR}/grpc-proto/grpc/testing/*.proto"
+  "${WORKDIR}"/grpc-proto/grpc/testing/*.proto
-  "${WORKDIR}/grpc-proto/grpc/core/*.proto"
+  "${WORKDIR}"/grpc-proto/grpc/core/*.proto
 )
 
 # These options of the form 'Mfoo.proto=bar' instruct the codegen to use an
@@ -94,9 +99,9 @@ Mgrpc/testing/test.proto=google.golang.org/grpc/interop/grpc_testing,\
 Mgrpc/testing/payloads.proto=google.golang.org/grpc/interop/grpc_testing,\
 Mgrpc/testing/empty.proto=google.golang.org/grpc/interop/grpc_testing
 
-for src in ${SOURCES[@]}; do
+for src in "${SOURCES[@]}"; do
   echo "protoc ${src}"
-  protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS}:${WORKDIR}/out \
+  protoc --go_out="${OPTS}:${WORKDIR}/out" --go-grpc_out="${OPTS}:${WORKDIR}/out" \
     -I"." \
     -I"${WORKDIR}/grpc-proto" \
     -I"${WORKDIR}/googleapis" \
@@ -104,9 +109,9 @@ for src in ${SOURCES[@]}; do
     "${src}"
 done
 
-for src in ${LEGACY_SOURCES[@]}; do
+for src in "${LEGACY_SOURCES[@]}"; do
   echo "protoc ${src}"
-  protoc --go_out="${OPTS}:${WORKDIR}/out" --go-grpc_out="${OPTS}",require_unimplemented_servers=false:"${WORKDIR}/out" \
+  protoc --go_out="${OPTS}:${WORKDIR}/out" --go-grpc_out="${OPTS},require_unimplemented_servers=false:${WORKDIR}/out" \
     -I"." \
     -I"${WORKDIR}/grpc-proto" \
     -I"${WORKDIR}/googleapis" \