buildscripts: make unix.sh and make_dependencies.sh arch aware (#4384)

ARCH can be '32' or '64'. If it is not set then default to '64'.

make_dependencies.sh should do the symlinking

.travis.yml

@@ -17,8 +17,7 @@ before_install:
     ln -s /tmp/gradle-wrapper $HOME/.gradle/wrapper
     # Work around https://github.com/travis-ci/travis-ci/issues/2317
   - if \[ "$TRAVIS_OS_NAME" = linux \]; then jdk_switcher use oraclejdk8; fi
-  - buildscripts/make_dependencies.sh # build protoc into /tmp/protobuf-${PROTOBUF_VERSION}
-  - ln -s "/tmp/protobuf-${PROTOBUF_VERSION}/$(uname -s)-$(uname -p)" /tmp/protobuf
+  - buildscripts/make_dependencies.sh # build protoc into /tmp/protobuf
   - mkdir -p $HOME/.gradle
   - echo "checkstyle.ignoreFailures=false" >> $HOME/.gradle/gradle.properties
   - echo "failOnWarnings=true" >> $HOME/.gradle/gradle.properties

buildscripts/kokoro/android.sh

@@ -19,7 +19,6 @@ export OS_NAME=$(uname)
 
 # Proto deps
 buildscripts/make_dependencies.sh
-ln -s "/tmp/protobuf-${PROTOBUF_VERSION}/$(uname -s)-$(uname -p)" /tmp/protobuf
 
 ./gradlew install
 

buildscripts/kokoro/unix.sh

@@ -1,12 +1,19 @@
 #!/bin/bash
 
 # This file is used for both Linux and MacOS builds.
+# For Linux, this script is called inside a docker container with
+# the correct environment for releases.
 # To run locally:
 #  ./buildscripts/kokoro/unix.sh
+# For 32 bit:
+#  ARCH=32 ./buildscripts/kokoro/unix.sh
 
 # This script assumes `set -e`. Removing it may lead to undefined behavior.
 set -exu -o pipefail
 
+# It would be nicer to use 'readlink -f' here but osx does not support it.
+readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)"
+
 if [[ -f /VERSION ]]; then
   cat /VERSION
 fi
@@ -19,26 +26,14 @@ cd $(dirname $0)/../..
 # Proto deps
 export PROTOBUF_VERSION=3.5.1
 
-# TODO(zpencer): if linux builds use this script, then also repeat this process for 32bit (-m32)
-# Today, only macos uses this script and macos targets 64bit only
+# ARCH is 64 bit unless otherwise specified.
+ARCH="${ARCH:-64}"
 
-CXX_FLAGS="-m64" LDFLAGS="" LD_LIBRARY_PATH="" buildscripts/make_dependencies.sh
-
-# the install dir is hardcoded in make_dependencies.sh
-PROTO_INSTALL_DIR="/tmp/protobuf-${PROTOBUF_VERSION}/$(uname -s)-$(uname -p)"
-
-if [[ ! -e /tmp/protobuf ]]; then
-  ln -s $PROTO_INSTALL_DIR /tmp/protobuf;
-fi
-
-# It's better to use 'readlink -f' but it's not available on macos
-if [[ "$(readlink /tmp/protobuf)" != "$PROTO_INSTALL_DIR" ]]; then
-  echo "/tmp/protobuf already exists but is not a symlink to $PROTO_INSTALL_DIR"
-  exit 1;
-fi
+ARCH="$ARCH" buildscripts/make_dependencies.sh
 
 # Set properties via flags, do not pollute gradle.properties
 GRADLE_FLAGS="${GRADLE_FLAGS:-}"
+GRADLE_FLAGS+=" -PtargetArch=x86_$ARCH $GRADLE_FLAGS"
 GRADLE_FLAGS+=" -Pcheckstyle.ignoreFailures=false"
 GRADLE_FLAGS+=" -PfailOnWarnings=true"
 GRADLE_FLAGS+=" -PerrorProne=true"
@@ -70,10 +65,10 @@ popd
 
 LOCAL_MVN_TEMP=$(mktemp -d)
 # Note that this disables parallel=true from GRADLE_FLAGS
-./gradlew clean grpc-compiler:build grpc-compiler:uploadArchives $GRADLE_FLAGS -PtargetArch=x86_64 \
-  -Dorg.gradle.parallel=false -PrepositoryDir=$LOCAL_MVN_TEMP
+./gradlew clean grpc-compiler:build grpc-compiler:uploadArchives $GRADLE_FLAGS \
+  -Dorg.gradle.parallel=false -PrepositoryDir="$LOCAL_MVN_TEMP"
 
-MVN_ARTIFACT_DIR="$PWD/mvn-artifacts"
-mkdir $MVN_ARTIFACT_DIR
-mv $LOCAL_MVN_TEMP/* $MVN_ARTIFACT_DIR
-rmdir $LOCAL_MVN_TEMP
+readonly MVN_ARTIFACT_DIR="${MVN_ARTIFACT_DIR:-$GRPC_JAVA_DIR/mvn-artifacts}"
+
+mkdir -p "$MVN_ARTIFACT_DIR"
+cp -r "$LOCAL_MVN_TEMP"/* "$MVN_ARTIFACT_DIR"/

buildscripts/make_dependencies.sh

@@ -3,8 +3,10 @@
 # Build protoc
 set -evux -o pipefail
 
+# ARCH is 64 bit unless otherwise specified.
+ARCH="${ARCH:-64}"
 DOWNLOAD_DIR=/tmp/source
-INSTALL_DIR="/tmp/protobuf-$PROTOBUF_VERSION/$(uname -s)-$(uname -p)"
+INSTALL_DIR="/tmp/protobuf-$PROTOBUF_VERSION/$(uname -s)-$(uname -p)-x86_$ARCH"
 mkdir -p $DOWNLOAD_DIR
 
 # Start with a sane default
@@ -22,13 +24,24 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then
   echo "Not building protobuf. Already built"
 # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date
 else
-  wget -O - https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.tar.gz | tar xz -C $DOWNLOAD_DIR
+  if [[ ! -d "$DOWNLOAD_DIR"/protobuf-"${PROTOBUF_VERSION}" ]]; then
+    wget -O - https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.tar.gz | tar xz -C $DOWNLOAD_DIR
+  fi
   pushd $DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION}
   ./autogen.sh
   # install here so we don't need sudo
-  ./configure --disable-shared --prefix="$INSTALL_DIR"
+  ./configure CFLAGS=-m"$ARCH" CXXFLAGS=-m"$ARCH" --disable-shared \
+    --prefix="$INSTALL_DIR"
+  # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first
+  make clean
   make -j$NUM_CPU
   make install
   popd
 fi
 
+# If /tmp/protobuf exists then we just assume it's a symlink created by us.
+# It may be that it points to the wrong arch, so we idempotently set it now.
+if [[ -L /tmp/protobuf ]]; then
+  rm /tmp/protobuf
+fi
+ln -s "$INSTALL_DIR" /tmp/protobuf