diff --git a/buildscripts/build_artifacts_in_docker.sh b/buildscripts/build_artifacts_in_docker.sh new file mode 100755 index 0000000000..ca24f24b32 --- /dev/null +++ b/buildscripts/build_artifacts_in_docker.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -exu -o pipefail + +# Runs all the tests and builds mvn artifacts. +# mvn artifacts are stored in grpc-java/mvn-artifacts/ +ALL_ARTIFACTS=true ARCH=64 "$(dirname $0)"/kokoro/unix.sh +# Already ran tests the first time, so skip tests this time +SKIP_TESTS=true ARCH=32 "$(dirname $0)"/kokoro/unix.sh diff --git a/buildscripts/grpc-java-releasing/Dockerfile b/buildscripts/grpc-java-releasing/Dockerfile index fe1a8687e3..3917a940f8 100644 --- a/buildscripts/grpc-java-releasing/Dockerfile +++ b/buildscripts/grpc-java-releasing/Dockerfile @@ -1,5 +1,8 @@ FROM protoc-artifacts:latest +# protoc-artifacts base image is ancient and curl does +# not work properly. Ignore rpmdb checksum errors from update. +RUN yum -y update; yum clean all # Update to >= 4/24/2018 COPY scl-enable-devtoolset.sh / # Start in devtoolset environment that uses GCC 4.7 diff --git a/buildscripts/kokoro/linux_artifacts.sh b/buildscripts/kokoro/linux_artifacts.sh old mode 100644 new mode 100755 index ebd72f6237..a97e8fae88 --- a/buildscripts/kokoro/linux_artifacts.sh +++ b/buildscripts/kokoro/linux_artifacts.sh @@ -5,6 +5,16 @@ if [[ -f /VERSION ]]; then cat /VERSION fi -readonly GRPC_JAVA_DIR=$(cd $(dirname $0)/../.. && pwd) +readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)" -# A place holder at the moment +rm -rf /tmp/protobuf/ +mkdir -p /tmp/protobuf/ +# Download an unreleased SHA because we need this fix: +# https://github.com/google/protobuf/pull/4447 +wget -O - https://github.com/google/protobuf/archive/92898e9e.tar.gz | tar xz -C /tmp/protobuf/ + +docker build -t protoc-artifacts /tmp/protobuf/protobuf-92898e9e9cb2f1c006fcc5099c9c96eafce63dc8/protoc-artifacts + +docker build -t grpc-java-releasing "$GRPC_JAVA_DIR"/buildscripts/grpc-java-releasing + +"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh /grpc-java/buildscripts/build_artifacts_in_docker.sh diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index b68327aee1..67bbe7f39e 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -45,35 +45,37 @@ export LD_LIBRARY_PATH=/tmp/protobuf/lib export LDFLAGS=-L/tmp/protobuf/lib export CXXFLAGS="-I/tmp/protobuf/include" -# Ensure all *.proto changes include *.java generated code -./gradlew assemble generateTestProto install $GRADLE_FLAGS +./gradlew clean $GRADLE_FLAGS -if [[ -z "${SKIP_CLEAN_CHECK:-}" && ! -z $(git status --porcelain) ]]; then - git status - echo "Error Working directory is not clean. Forget to commit generated files?" - exit 1 +if [[ -z "${SKIP_TESTS:-}" ]]; then + # Ensure all *.proto changes include *.java generated code + ./gradlew assemble generateTestProto install $GRADLE_FLAGS + + if [[ -z "${SKIP_CLEAN_CHECK:-}" && ! -z $(git status --porcelain) ]]; then + git status + echo "Error Working directory is not clean. Forget to commit generated files?" + exit 1 + fi + # Run tests + ./gradlew build $GRADLE_FLAGS + pushd examples + ./gradlew clean $GRADLE_FLAGS + ./gradlew build $GRADLE_FLAGS + # --batch-mode reduces log spam + mvn clean verify --batch-mode + popd + # TODO(zpencer): also build the GAE examples fi -# Run tests -./gradlew build $GRADLE_FLAGS -./gradlew install - -pushd examples -./gradlew build $GRADLE_FLAGS -# --batch-mode reduces log spam -mvn verify --batch-mode -popd - -pushd examples/example-kotlin/ -./gradlew build $GRADLE_FLAGS -popd - -# TODO(zpencer): also build the GAE examples - LOCAL_MVN_TEMP=$(mktemp -d) # Note that this disables parallel=true from GRADLE_FLAGS -./gradlew clean grpc-compiler:build grpc-compiler:uploadArchives $GRADLE_FLAGS \ - -Dorg.gradle.parallel=false -PrepositoryDir="$LOCAL_MVN_TEMP" +if [[ -z "${ALL_ARTIFACTS:-}" ]]; then + ./gradlew grpc-compiler:build grpc-compiler:uploadArchives $GRADLE_FLAGS \ + -Dorg.gradle.parallel=false -PrepositoryDir=$LOCAL_MVN_TEMP +else + ./gradlew uploadArchives $GRADLE_FLAGS \ + -Dorg.gradle.parallel=false -PrepositoryDir=$LOCAL_MVN_TEMP +fi readonly MVN_ARTIFACT_DIR="${MVN_ARTIFACT_DIR:-$GRPC_JAVA_DIR/mvn-artifacts}" diff --git a/buildscripts/kokoro/upload_artifacts.cfg b/buildscripts/kokoro/upload_artifacts.cfg new file mode 100644 index 0000000000..38ee2f38ae --- /dev/null +++ b/buildscripts/kokoro/upload_artifacts.cfg @@ -0,0 +1,5 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-java/buildscripts/kokoro/upload_artifacts.sh" +timeout_mins: 60 diff --git a/buildscripts/kokoro/upload_artifacts.sh b/buildscripts/kokoro/upload_artifacts.sh new file mode 100644 index 0000000000..ff29fcb0fc --- /dev/null +++ b/buildscripts/kokoro/upload_artifacts.sh @@ -0,0 +1,57 @@ +#!/bin/bash +set -veux -o pipefail + +if [[ -f /VERSION ]]; then + cat /VERSION +fi + +readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)" + +echo "Verifying that all artifacts are here..." +find "$KOKORO_GFILE_DIR" + +# verify that files from all 3 grouped jobs are present. +# platform independent artifacts, from linux job: +[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'grpc-core-*.jar' | wc -l)" != '0' ]] + +# from linux job: +[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'protoc-gen-grpc-java-*-linux-x86_64.exe' | wc -l)" != '0' ]] +[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'protoc-gen-grpc-java-*-linux-x86_32.exe' | wc -l)" != '0' ]] + +# from macos job: +[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'protoc-gen-grpc-java-*-osx-x86_64.exe' | wc -l)" != '0' ]] + +# from windows job: +[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'protoc-gen-grpc-java-*-windows-x86_64.exe' | wc -l)" != '0' ]] +[[ "$(find "$KOKORO_GFILE_DIR" -type f -iname 'protoc-gen-grpc-java-*-windows-x86_32.exe' | wc -l)" != '0' ]] + + +mkdir -p ~/.config/ +gsutil cp gs://grpc-testing-secrets/sonatype_credentials/sonatype-upload ~/.config/sonatype-upload + +mkdir -p ~/java_signing/ +gsutil cp -r gs://grpc-testing-secrets/java_signing/ ~/ +gpg --batch --import ~/java_signing/grpc-java-team-sonatype.asc + +# gpg commands changed between v1 and v2 are different. +gpg --version + +# This is the version found on kokoro. +if gpg --version | grep 'gpg (GnuPG) 1.'; then + # This command was tested on 1.4.16 + find "$KOKORO_GFILE_DIR" -type f -exec \ + bash -c 'cat ~/java_signing/passphrase | gpg --batch --passphrase-fd 0 --detach-sign -o {}.asc {}' \; +fi + +# This is the version found on corp workstations. Maybe kokoro will be updated to gpg2 some day. +if gpg --version | grep 'gpg (GnuPG) 2.'; then + # This command was tested on 2.2.2 + find "$KOKORO_GFILE_DIR" -type f -exec \ + gpg --batch --passphrase-file ~/java_signing/passphrase --pinentry-mode loopback \ + --detach-sign -o {}.asc {} \; +fi + +STAGING_REPO=a93898609ef848 +find "$KOKORO_GFILE_DIR" -name 'mvn-artifacts' -type d -exec \ + "$GRPC_JAVA_DIR"/buildscripts/sonatype-upload.sh \ + "$STAGING_REPO" {} \; diff --git a/buildscripts/kokoro/windows.bat b/buildscripts/kokoro/windows.bat index 6a1c508c94..709b61c988 100644 --- a/buildscripts/kokoro/windows.bat +++ b/buildscripts/kokoro/windows.bat @@ -21,5 +21,4 @@ cmd.exe /C "%WORKSPACE%\buildscripts\kokoro\windows32.bat" || exit /b 1 cmd.exe /C "%WORKSPACE%\buildscripts\kokoro\windows64.bat" || exit /b 1 mkdir mvn-artifacts -move artifacts\x86_64 mvn-artifacts\x86_64 -move artifacts\x86_32 mvn-artifacts\x86_32 +move artifacts\io mvn-artifacts diff --git a/buildscripts/kokoro/windows32.bat b/buildscripts/kokoro/windows32.bat index a03a220a2a..dbe677d697 100644 --- a/buildscripts/kokoro/windows32.bat +++ b/buildscripts/kokoro/windows32.bat @@ -48,4 +48,4 @@ IF NOT %GRADLEEXIT% == 0 ( @rem make sure no daemons have any files open cmd.exe /C "%WORKSPACE%\gradlew.bat --stop" -cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts\x86_32\ clean grpc-compiler:build grpc-compiler:uploadArchives" || exit /b 1 +cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts clean grpc-compiler:build grpc-compiler:uploadArchives" || exit /b 1 diff --git a/buildscripts/kokoro/windows64.bat b/buildscripts/kokoro/windows64.bat index 43ee96c4b8..6346b44d1b 100644 --- a/buildscripts/kokoro/windows64.bat +++ b/buildscripts/kokoro/windows64.bat @@ -32,4 +32,4 @@ SET GRADLE_FLAGS=-PtargetArch=%TARGET_ARCH% -PfailOnWarnings=%FAIL_ON_WARNINGS% @rem make sure no daemons have any files open cmd.exe /C "%WORKSPACE%\gradlew.bat --stop" -cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts\x86_64\ clean grpc-compiler:build grpc-compiler:uploadArchives" || exit /b 1 +cmd.exe /C "%WORKSPACE%\gradlew.bat %GRADLE_FLAGS% -Dorg.gradle.parallel=false -PrepositoryDir=%WORKSPACE%\artifacts clean grpc-compiler:build grpc-compiler:uploadArchives" || exit /b 1 diff --git a/buildscripts/run_in_docker.sh b/buildscripts/run_in_docker.sh index f6ec7e32c6..616ecf6ce1 100755 --- a/buildscripts/run_in_docker.sh +++ b/buildscripts/run_in_docker.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eu -o pipefail +set -eux -o pipefail quote() { local arg @@ -10,31 +10,16 @@ quote() { done } -if [[ "${1:-}" != "in-docker" ]]; then - readonly grpc_java_dir="$(dirname $(readlink -f "$0"))/.." - exec docker run -it --rm=true -v "${grpc_java_dir}:/grpc-java" -w /grpc-java \ - grpc-java-releasing \ - ./buildscripts/run_in_docker.sh in-docker "$(id -u)" "$(id -g)" "$@" -fi - -## In Docker - -shift - -readonly swap_uid="$1" -readonly swap_gid="$2" -shift 2 - -# Java uses NSS to determine the user's home. If that fails it uses '?' in the -# current directory. So we need to set up the user's home in /etc/passwd. -# If this wasn't the case, we could have passed -u to docker run and avoided -# this script inside the container. JAVA_TOOL_OPTIONS is okay, but is noisy. -groupadd thegroup -g "$swap_gid" -useradd theuser -u "$swap_uid" -g "$swap_gid" -m -if [[ "$#" -eq 0 ]]; then - exec su theuser +readonly grpc_java_dir="$(dirname "$(readlink -f "$0")")/.." +if [[ -t 0 ]]; then + DOCKER_ARGS="-it" else - # runuser is too old in the container to support the -u flag; if it did, we'd - # be able to remove the 'quote' function. - exec su theuser -c "$(quote "$@")" + # The input device on kokoro is not a TTY, so -it does not work. + DOCKER_ARGS= fi +# Use a trap function to fix file permissions upon exit, without affecting +# the original exit code. $DOCKER_ARGS can not be quoted, otherwise it becomes a '' which confuses +# docker. +exec docker run $DOCKER_ARGS --rm=true -v "${grpc_java_dir}":/grpc-java -w /grpc-java \ + grpc-java-releasing \ + bash -c "function fixFiles() { chown -R $(id -u):$(id -g) /grpc-java; }; trap fixFiles EXIT; $(quote "$@")"