From 1d80febbc0090e7333717b79daaeb28cc3160255 Mon Sep 17 00:00:00 2001 From: zpencer Date: Fri, 27 Apr 2018 15:59:16 -0700 Subject: [PATCH] buildscripts: make unix.sh and make_dependencies.sh arch aware (#4384) ARCH can be '32' or '64'. If it is not set then default to '64'. make_dependencies.sh should do the symlinking --- .travis.yml | 3 +-- buildscripts/kokoro/android.sh | 1 - buildscripts/kokoro/unix.sh | 39 ++++++++++++++----------------- buildscripts/make_dependencies.sh | 19 ++++++++++++--- 4 files changed, 34 insertions(+), 28 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7ea9aee3d9..9ed002356c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,7 @@ before_install: ln -s /tmp/gradle-wrapper $HOME/.gradle/wrapper # Work around https://github.com/travis-ci/travis-ci/issues/2317 - if \[ "$TRAVIS_OS_NAME" = linux \]; then jdk_switcher use oraclejdk8; fi - - buildscripts/make_dependencies.sh # build protoc into /tmp/protobuf-${PROTOBUF_VERSION} - - ln -s "/tmp/protobuf-${PROTOBUF_VERSION}/$(uname -s)-$(uname -p)" /tmp/protobuf + - buildscripts/make_dependencies.sh # build protoc into /tmp/protobuf - mkdir -p $HOME/.gradle - echo "checkstyle.ignoreFailures=false" >> $HOME/.gradle/gradle.properties - echo "failOnWarnings=true" >> $HOME/.gradle/gradle.properties diff --git a/buildscripts/kokoro/android.sh b/buildscripts/kokoro/android.sh index dd50d448f4..a52ab10400 100755 --- a/buildscripts/kokoro/android.sh +++ b/buildscripts/kokoro/android.sh @@ -19,7 +19,6 @@ export OS_NAME=$(uname) # Proto deps buildscripts/make_dependencies.sh -ln -s "/tmp/protobuf-${PROTOBUF_VERSION}/$(uname -s)-$(uname -p)" /tmp/protobuf ./gradlew install diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh index ff0f580fb4..afcb7793e3 100755 --- a/buildscripts/kokoro/unix.sh +++ b/buildscripts/kokoro/unix.sh @@ -1,12 +1,19 @@ #!/bin/bash # This file is used for both Linux and MacOS builds. +# For Linux, this script is called inside a docker container with +# the correct environment for releases. # To run locally: # ./buildscripts/kokoro/unix.sh +# For 32 bit: +# ARCH=32 ./buildscripts/kokoro/unix.sh # This script assumes `set -e`. Removing it may lead to undefined behavior. set -exu -o pipefail +# It would be nicer to use 'readlink -f' here but osx does not support it. +readonly GRPC_JAVA_DIR="$(cd "$(dirname "$0")"/../.. && pwd)" + if [[ -f /VERSION ]]; then cat /VERSION fi @@ -19,26 +26,14 @@ cd $(dirname $0)/../.. # Proto deps export PROTOBUF_VERSION=3.5.1 -# TODO(zpencer): if linux builds use this script, then also repeat this process for 32bit (-m32) -# Today, only macos uses this script and macos targets 64bit only +# ARCH is 64 bit unless otherwise specified. +ARCH="${ARCH:-64}" -CXX_FLAGS="-m64" LDFLAGS="" LD_LIBRARY_PATH="" buildscripts/make_dependencies.sh - -# the install dir is hardcoded in make_dependencies.sh -PROTO_INSTALL_DIR="/tmp/protobuf-${PROTOBUF_VERSION}/$(uname -s)-$(uname -p)" - -if [[ ! -e /tmp/protobuf ]]; then - ln -s $PROTO_INSTALL_DIR /tmp/protobuf; -fi - -# It's better to use 'readlink -f' but it's not available on macos -if [[ "$(readlink /tmp/protobuf)" != "$PROTO_INSTALL_DIR" ]]; then - echo "/tmp/protobuf already exists but is not a symlink to $PROTO_INSTALL_DIR" - exit 1; -fi +ARCH="$ARCH" buildscripts/make_dependencies.sh # Set properties via flags, do not pollute gradle.properties GRADLE_FLAGS="${GRADLE_FLAGS:-}" +GRADLE_FLAGS+=" -PtargetArch=x86_$ARCH $GRADLE_FLAGS" GRADLE_FLAGS+=" -Pcheckstyle.ignoreFailures=false" GRADLE_FLAGS+=" -PfailOnWarnings=true" GRADLE_FLAGS+=" -PerrorProne=true" @@ -70,10 +65,10 @@ popd LOCAL_MVN_TEMP=$(mktemp -d) # Note that this disables parallel=true from GRADLE_FLAGS -./gradlew clean grpc-compiler:build grpc-compiler:uploadArchives $GRADLE_FLAGS -PtargetArch=x86_64 \ - -Dorg.gradle.parallel=false -PrepositoryDir=$LOCAL_MVN_TEMP +./gradlew clean grpc-compiler:build grpc-compiler:uploadArchives $GRADLE_FLAGS \ + -Dorg.gradle.parallel=false -PrepositoryDir="$LOCAL_MVN_TEMP" -MVN_ARTIFACT_DIR="$PWD/mvn-artifacts" -mkdir $MVN_ARTIFACT_DIR -mv $LOCAL_MVN_TEMP/* $MVN_ARTIFACT_DIR -rmdir $LOCAL_MVN_TEMP +readonly MVN_ARTIFACT_DIR="${MVN_ARTIFACT_DIR:-$GRPC_JAVA_DIR/mvn-artifacts}" + +mkdir -p "$MVN_ARTIFACT_DIR" +cp -r "$LOCAL_MVN_TEMP"/* "$MVN_ARTIFACT_DIR"/ diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh index 7a0fe5027d..41cb406eac 100755 --- a/buildscripts/make_dependencies.sh +++ b/buildscripts/make_dependencies.sh @@ -3,8 +3,10 @@ # Build protoc set -evux -o pipefail +# ARCH is 64 bit unless otherwise specified. +ARCH="${ARCH:-64}" DOWNLOAD_DIR=/tmp/source -INSTALL_DIR="/tmp/protobuf-$PROTOBUF_VERSION/$(uname -s)-$(uname -p)" +INSTALL_DIR="/tmp/protobuf-$PROTOBUF_VERSION/$(uname -s)-$(uname -p)-x86_$ARCH" mkdir -p $DOWNLOAD_DIR # Start with a sane default @@ -22,13 +24,24 @@ if [ -f ${INSTALL_DIR}/bin/protoc ]; then echo "Not building protobuf. Already built" # TODO(ejona): swap to `brew install --devel protobuf` once it is up-to-date else - wget -O - https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.tar.gz | tar xz -C $DOWNLOAD_DIR + if [[ ! -d "$DOWNLOAD_DIR"/protobuf-"${PROTOBUF_VERSION}" ]]; then + wget -O - https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.tar.gz | tar xz -C $DOWNLOAD_DIR + fi pushd $DOWNLOAD_DIR/protobuf-${PROTOBUF_VERSION} ./autogen.sh # install here so we don't need sudo - ./configure --disable-shared --prefix="$INSTALL_DIR" + ./configure CFLAGS=-m"$ARCH" CXXFLAGS=-m"$ARCH" --disable-shared \ + --prefix="$INSTALL_DIR" + # the same source dir is used for 32 and 64 bit builds, so we need to clean stale data first + make clean make -j$NUM_CPU make install popd fi +# If /tmp/protobuf exists then we just assume it's a symlink created by us. +# It may be that it points to the wrong arch, so we idempotently set it now. +if [[ -L /tmp/protobuf ]]; then + rm /tmp/protobuf +fi +ln -s "$INSTALL_DIR" /tmp/protobuf