buildscripts: automate the release process (#4406)

These scripts set up the jobs needed to do one click releases.

List of changes:

Do not put files in mvn-artifacts/$ARCH. This makes the view uniform
across unix and windows and is easier to sign and upload.

A working curl is needed to build protobuf from source, so run yum
update.

run_in_docker.sh: use chmod to fix permisisons, do not set up user

linux_artifact builds 32 and 64 artifacts using unix.sh

add upload_artifacts cfg and script

buildscripts/build_artifacts_in_docker.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -exu -o pipefail
+
+# Runs all the tests and builds mvn artifacts.
+# mvn artifacts are stored in grpc-java/mvn-artifacts/
+ALL_ARTIFACTS=true ARCH=64 "$(dirname $0)"/kokoro/unix.sh
+# Already ran tests the first time, so skip tests this time
+SKIP_TESTS=true ARCH=32 "$(dirname $0)"/kokoro/unix.sh

buildscripts/grpc-java-releasing/Dockerfile

@@ -1,5 +1,8 @@
 FROM protoc-artifacts:latest
 
+# protoc-artifacts base image is ancient and curl does
+# not work properly. Ignore rpmdb checksum errors from update.
+RUN yum -y update; yum clean all # Update to >= 4/24/2018
 COPY scl-enable-devtoolset.sh /
 
 # Start in devtoolset environment that uses GCC 4.7

buildscripts/kokoro/linux_artifacts.sh

@@ -5,6 +5,16 @@ if [[ -f /VERSION ]]; then
   cat /VERSION
 fi
 
-readonly GRPC_JAVA_DIR=$(cd $(dirname $0)/../.. && pwd)
+readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)"
 
-# A place holder at the moment
+rm -rf /tmp/protobuf/
+mkdir -p /tmp/protobuf/
+# Download an unreleased SHA because we need this fix:
+# https://github.com/google/protobuf/pull/4447
+wget -O - https://github.com/google/protobuf/archive/92898e9e.tar.gz | tar xz -C /tmp/protobuf/
+
+docker build -t protoc-artifacts /tmp/protobuf/protobuf-92898e9e9cb2f1c006fcc5099c9c96eafce63dc8/protoc-artifacts
+
+docker build -t grpc-java-releasing "$GRPC_JAVA_DIR"/buildscripts/grpc-java-releasing
+
+"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh /grpc-java/buildscripts/build_artifacts_in_docker.sh

buildscripts/kokoro/unix.sh

@@ -45,35 +45,37 @@ export LD_LIBRARY_PATH=/tmp/protobuf/lib
 export LDFLAGS=-L/tmp/protobuf/lib
 export CXXFLAGS="-I/tmp/protobuf/include"
 
-# Ensure all *.proto changes include *.java generated code
+./gradlew clean $GRADLE_FLAGS
-./gradlew assemble generateTestProto install $GRADLE_FLAGS
 
-if [[ -z "${SKIP_CLEAN_CHECK:-}" && ! -z $(git status --porcelain) ]]; then
+if [[ -z "${SKIP_TESTS:-}" ]]; then
-  git status
+  # Ensure all *.proto changes include *.java generated code
-  echo "Error Working directory is not clean. Forget to commit generated files?"
+  ./gradlew assemble generateTestProto install $GRADLE_FLAGS
-  exit 1
+
+  if [[ -z "${SKIP_CLEAN_CHECK:-}" && ! -z $(git status --porcelain) ]]; then
+    git status
+    echo "Error Working directory is not clean. Forget to commit generated files?"
+    exit 1
+  fi
+  # Run tests
+  ./gradlew build $GRADLE_FLAGS
+  pushd examples
+  ./gradlew clean $GRADLE_FLAGS
+  ./gradlew build $GRADLE_FLAGS
+  # --batch-mode reduces log spam
+  mvn clean verify --batch-mode
+  popd
+  # TODO(zpencer): also build the GAE examples
 fi
 
-# Run tests
-./gradlew build $GRADLE_FLAGS
-./gradlew install
-
-pushd examples
-./gradlew build $GRADLE_FLAGS
-# --batch-mode reduces log spam
-mvn verify --batch-mode
-popd
-
-pushd examples/example-kotlin/
-./gradlew build $GRADLE_FLAGS
-popd
-
-# TODO(zpencer): also build the GAE examples
-
 LOCAL_MVN_TEMP=$(mktemp -d)
 # Note that this disables parallel=true from GRADLE_FLAGS
-./gradlew clean grpc-compiler:build grpc-compiler:uploadArchives $GRADLE_FLAGS \
+if [[ -z "${ALL_ARTIFACTS:-}" ]]; then
-  -Dorg.gradle.parallel=false -PrepositoryDir="$LOCAL_MVN_TEMP"
+  ./gradlew grpc-compiler:build grpc-compiler:uploadArchives $GRADLE_FLAGS \
+    -Dorg.gradle.parallel=false -PrepositoryDir=$LOCAL_MVN_TEMP
+else
+  ./gradlew uploadArchives $GRADLE_FLAGS \
+    -Dorg.gradle.parallel=false -PrepositoryDir=$LOCAL_MVN_TEMP
+fi
 
 readonly MVN_ARTIFACT_DIR="${MVN_ARTIFACT_DIR:-$GRPC_JAVA_DIR/mvn-artifacts}"
 

buildscripts/kokoro/upload_artifacts.cfg

@@ -0,0 +1,5 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-java/buildscripts/kokoro/upload_artifacts.sh"
+timeout_mins: 60

buildscripts/kokoro/upload_artifacts.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -veux -o pipefail
+
+if [[ -f /VERSION ]]; then
+  cat /VERSION
+fi
+
+readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)"
+
+echo "Verifying that all artifacts are here..."
+find "$KOKORO_GFILE_DIR"
+
+# verify that files from all 3 grouped jobs are present.
+# platform independent artifacts, from linux job:
+[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'grpc-core-*.jar' | wc -l)" != '0' ]]
+
+# from linux job:
+[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'protoc-gen-grpc-java-*-linux-x86_64.exe' | wc -l)" != '0' ]]
+[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'protoc-gen-grpc-java-*-linux-x86_32.exe' | wc -l)" != '0' ]]
+
+# from macos job:
+[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'protoc-gen-grpc-java-*-osx-x86_64.exe' | wc -l)" != '0' ]]
+
+# from windows job:
+[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'protoc-gen-grpc-java-*-windows-x86_64.exe' | wc -l)" != '0' ]]
+[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'protoc-gen-grpc-java-*-windows-x86_32.exe' | wc -l)" != '0' ]]
+
+
+mkdir -p ~/.config/
+gsutil cp gs://grpc-testing-secrets/sonatype_credentials/sonatype-upload ~/.config/sonatype-upload
+
+mkdir -p ~/java_signing/
+gsutil cp -r gs://grpc-testing-secrets/java_signing/ ~/
+gpg --batch  --import ~/java_signing/grpc-java-team-sonatype.asc
+
+# gpg commands changed between v1 and v2 are different.
+gpg --version
+
+# This is the version found on kokoro.
+if gpg --version | grep 'gpg (GnuPG) 1.'; then
+  # This command was tested on 1.4.16
+  find "$KOKORO_GFILE_DIR" -type f -exec \
+  bash -c 'cat ~/java_signing/passphrase | gpg --batch --passphrase-fd 0 --detach-sign -o {}.asc {}' \;
+fi
+
+# This is the version found on corp workstations. Maybe kokoro will be updated to gpg2 some day.
+if gpg --version | grep 'gpg (GnuPG) 2.'; then
+  # This command was tested on 2.2.2
+  find "$KOKORO_GFILE_DIR" -type f -exec \
+    gpg --batch --passphrase-file ~/java_signing/passphrase --pinentry-mode loopback \
+    --detach-sign -o {}.asc {} \;
+fi
+
+STAGING_REPO=a93898609ef848
+find "$KOKORO_GFILE_DIR" -name 'mvn-artifacts' -type d -exec \
+  "$GRPC_JAVA_DIR"/buildscripts/sonatype-upload.sh \
+  "$STAGING_REPO" {} \;

buildscripts/kokoro/windows.bat

@@ -21,5 +21,4 @@ cmd.exe /C "%WORKSPACE%\buildscripts\kokoro\windows32.bat" || exit /b 1
 cmd.exe /C "%WORKSPACE%\buildscripts\kokoro\windows64.bat" || exit /b 1
 
 mkdir mvn-artifacts
-move artifacts\x86_64 mvn-artifacts\x86_64
+move artifacts\io mvn-artifacts
-move artifacts\x86_32 mvn-artifacts\x86_32

buildscripts/kokoro/windows32.bat

@@ -48,4 +48,4 @@ IF NOT %GRADLEEXIT% == 0 (
 @rem make sure no daemons have any files open
 cmd.exe /C "%WORKSPACE%\gradlew.bat --stop"
 
-cmd.exe /C "%WORKSPACE%\gradlew.bat  %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts\x86_32\ clean grpc-compiler:build grpc-compiler:uploadArchives" || exit /b 1
+cmd.exe /C "%WORKSPACE%\gradlew.bat  %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts clean grpc-compiler:build grpc-compiler:uploadArchives" || exit /b 1

buildscripts/kokoro/windows64.bat

@@ -32,4 +32,4 @@ SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS%
 @rem make sure no daemons have any files open
 cmd.exe /C "%WORKSPACE%\gradlew.bat --stop"
 
-cmd.exe /C "%WORKSPACE%\gradlew.bat  %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts\x86_64\ clean grpc-compiler:build grpc-compiler:uploadArchives" || exit /b 1
+cmd.exe /C "%WORKSPACE%\gradlew.bat  %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts clean grpc-compiler:build grpc-compiler:uploadArchives" || exit /b 1

buildscripts/run_in_docker.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -eu -o pipefail
+set -eux -o pipefail
 
 quote() {
   local arg
@@ -10,31 +10,16 @@ quote() {
   done
 }
 
-if [[ "${1:-}" != "in-docker" ]]; then
+readonly grpc_java_dir="$(dirname "$(readlink -f "$0")")/.."
-  readonly grpc_java_dir="$(dirname $(readlink -f "$0"))/.."
+if [[ -t 0 ]]; then
-  exec docker run -it --rm=true -v "${grpc_java_dir}:/grpc-java" -w /grpc-java \
+  DOCKER_ARGS="-it"
-    grpc-java-releasing \
-    ./buildscripts/run_in_docker.sh in-docker "$(id -u)" "$(id -g)" "$@"
-fi
-
-## In Docker
-
-shift
-
-readonly swap_uid="$1"
-readonly swap_gid="$2"
-shift 2
-
-# Java uses NSS to determine the user's home. If that fails it uses '?' in the
-# current directory. So we need to set up the user's home in /etc/passwd.
-# If this wasn't the case, we could have passed -u to docker run and avoided
-# this script inside the container. JAVA_TOOL_OPTIONS is okay, but is noisy.
-groupadd thegroup -g "$swap_gid"
-useradd theuser -u "$swap_uid" -g "$swap_gid" -m
-if [[ "$#" -eq 0 ]]; then
-  exec su theuser
 else
-  # runuser is too old in the container to support the -u flag; if it did, we'd
+  # The input device on kokoro is not a TTY, so -it does not work.
-  # be able to remove the 'quote' function.
+  DOCKER_ARGS=
-  exec su theuser -c "$(quote "$@")"
 fi
+# Use a trap function to fix file permissions upon exit, without affecting
+# the original exit code. $DOCKER_ARGS can not be quoted, otherwise it becomes a '' which confuses
+# docker.
+exec docker run $DOCKER_ARGS --rm=true -v "${grpc_java_dir}":/grpc-java -w /grpc-java \
+  grpc-java-releasing \
+  bash -c "function fixFiles() { chown -R $(id -u):$(id -g) /grpc-java; }; trap fixFiles EXIT; $(quote "$@")"