From 383af9d1ba4ef9367ebab428f9b47916da9c0fb8 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 20 Oct 2017 10:09:23 -0700 Subject: [PATCH 1/7] Add artifact build scripts for native core --- packages/grpc-native-core/binding.gyp | 2 +- .../templates/binding.gyp.template | 2 +- .../artifacts/build_artifact_node.bat | 48 +++++++++++ .../artifacts/build_artifact_node.sh | 51 +++++++++++ .../run_tests/artifacts/build_package_node.sh | 84 +++++++++++++++++++ 5 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.bat create mode 100755 packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.sh create mode 100755 packages/grpc-native-core/tools/run_tests/artifacts/build_package_node.sh diff --git a/packages/grpc-native-core/binding.gyp b/packages/grpc-native-core/binding.gyp index f35bd619..9c8646c0 100644 --- a/packages/grpc-native-core/binding.gyp +++ b/packages/grpc-native-core/binding.gyp @@ -208,7 +208,7 @@ 'product_prefix': 'lib', 'type': 'static_library', 'cflags': [ - '-Wimplicit-fallthrough=0' + '-Wno-implicit-fallthrough' ], 'dependencies': [ ], diff --git a/packages/grpc-native-core/templates/binding.gyp.template b/packages/grpc-native-core/templates/binding.gyp.template index 19f88683..6385cd4e 100644 --- a/packages/grpc-native-core/templates/binding.gyp.template +++ b/packages/grpc-native-core/templates/binding.gyp.template @@ -195,7 +195,7 @@ 'product_prefix': 'lib', 'type': 'static_library', 'cflags': [ - '-Wimplicit-fallthrough=0' + '-Wno-implicit-fallthrough' ], 'dependencies': [ % for dep in getattr(lib, 'deps', []): diff --git a/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.bat b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.bat new file mode 100644 index 00000000..990ed7f4 --- /dev/null +++ b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.bat @@ -0,0 +1,48 @@ +@rem Copyright 2016 gRPC authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +set node_versions=4.0.0 5.0.0 6.0.0 7.0.0 8.0.0 + +set electron_versions=1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 1.7.0 + +set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm + +del /f /q BUILD || rmdir build /s /q + +call npm update || goto :error + +mkdir -p %ARTIFACTS_OUT% + +for %%v in (%node_versions%) do ( + call .\node_modules\.bin\node-pre-gyp.cmd configure build --target=%%v --target_arch=%1 + +@rem Try again after removing openssl headers + rmdir "%USERPROFILE%\.node-gyp\%%v\include\node\openssl" /S /Q + rmdir "%USERPROFILE%\.node-gyp\iojs-%%v\include\node\openssl" /S /Q + call .\node_modules\.bin\node-pre-gyp.cmd build package --target=%%v --target_arch=%1 || goto :error + + xcopy /Y /I /S build\stage\* %ARTIFACTS_OUT%\ || goto :error +) + +for %%v in (%electron_versions%) do ( + cmd /V /C "set "HOME=%USERPROFILE%\electron-gyp" && call .\node_modules\.bin\node-pre-gyp.cmd configure rebuild package --runtime=electron --target=%%v --target_arch=%1 --disturl=https://atom.io/download/electron" || goto :error + + xcopy /Y /I /S build\stage\* %ARTIFACTS_OUT%\ || goto :error +) +if %errorlevel% neq 0 exit /b %errorlevel% + +goto :EOF + +:error +exit /b 1 diff --git a/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.sh b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.sh new file mode 100755 index 00000000..5669fe29 --- /dev/null +++ b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +NODE_TARGET_ARCH=$1 +NODE_TARGET_OS=$2 +source ~/.nvm/nvm.sh + +nvm use 8 +set -ex + +cd $(dirname $0)/../../.. + +rm -rf build || true + +mkdir -p "${ARTIFACTS_OUT}" + +npm update + +node_versions=( 4.0.0 5.0.0 6.0.0 7.0.0 8.0.0 ) + +electron_versions=( 1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 1.7.0 ) + +for version in ${node_versions[@]} +do + ./node_modules/.bin/node-pre-gyp configure rebuild package --target=$version --target_arch=$NODE_TARGET_ARCH --grpc_alpine=true + cp -r build/stage/* "${ARTIFACTS_OUT}"/ + if [ "$NODE_TARGET_ARCH" == 'x64' ] && [ "$NODE_TARGET_OS" == 'linux' ] + then + # Cross compile for ARM on x64 + CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++ LD=arm-linux-gnueabihf-g++ ./node_modules/.bin/node-pre-gyp configure rebuild package testpackage --target=$version --target_arch=arm + cp -r build/stage/* "${ARTIFACTS_OUT}"/ + fi +done + +for version in ${electron_versions[@]} +do + HOME=~/.electron-gyp ./node_modules/.bin/node-pre-gyp configure rebuild package --runtime=electron --target=$version --target_arch=$NODE_TARGET_ARCH --disturl=https://atom.io/download/electron + cp -r build/stage/* "${ARTIFACTS_OUT}"/ +done diff --git a/packages/grpc-native-core/tools/run_tests/artifacts/build_package_node.sh b/packages/grpc-native-core/tools/run_tests/artifacts/build_package_node.sh new file mode 100755 index 00000000..2860f68b --- /dev/null +++ b/packages/grpc-native-core/tools/run_tests/artifacts/build_package_node.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright 2016 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source ~/.nvm/nvm.sh + +nvm use 8 +set -ex + +cd $(dirname $0)/../../.. + +base=$(pwd) + +artifacts=$base/artifacts + +mkdir -p $artifacts +cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/node_ext_*/* $artifacts/ || true + +npm update +npm pack + +cp grpc-*.tgz $artifacts/grpc.tgz + +mkdir -p bin + +cd $base/src/node/health_check +npm pack +cp grpc-health-check-*.tgz $artifacts/ + +cd $base/src/node/tools +npm update +npm pack +cp grpc-tools-*.tgz $artifacts/ +tools_version=$(npm list | grep -oP '(?<=grpc-tools@)\S+') + +output_dir=$artifacts/grpc-precompiled-binaries/node/grpc-tools/v$tools_version +mkdir -p $output_dir + +well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers ) + +for arch in {x86,x64}; do + case $arch in + x86) + node_arch=ia32 + ;; + *) + node_arch=$arch + ;; + esac + for plat in {windows,linux,macos}; do + case $plat in + windows) + node_plat=win32 + ;; + macos) + node_plat=darwin + ;; + *) + node_plat=$plat + ;; + esac + rm -r bin/* + input_dir="$EXTERNAL_GIT_ROOT/platform=${plat}/artifacts/protoc_${plat}_${arch}" + cp $input_dir/protoc* bin/ + cp $input_dir/grpc_node_plugin* bin/ + mkdir -p bin/google/protobuf + mkdir -p bin/google/protobuf/compiler # needed for plugin.proto + for proto in "${well_known_protos[@]}"; do + cp $base/third_party/protobuf/src/google/protobuf/$proto.proto bin/google/protobuf/$proto.proto + done + tar -czf $output_dir/$node_plat-$node_arch.tar.gz bin/ + done +done From 38637405dfac7bf3ba709c73b39db7ef51c5cdc3 Mon Sep 17 00:00:00 2001 From: Thom Wright Date: Tue, 31 Oct 2017 14:12:22 +0000 Subject: [PATCH 2/7] Include index.d.ts in the published npm package index.d.ts needs to be in the files list in order to be published. Without it, client packages aren't able to use the Typescript types. --- packages/grpc-native-core/package.json | 5 +++-- packages/grpc-native-core/templates/package.json.template | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/grpc-native-core/package.json b/packages/grpc-native-core/package.json index 3704c591..227c4a08 100644 --- a/packages/grpc-native-core/package.json +++ b/packages/grpc-native-core/package.json @@ -16,7 +16,7 @@ } ], "directories": { - "lib": "src/node/src" + "lib": "src" }, "scripts": { "build": "./node_modules/.bin/node-pre-gyp build", @@ -53,6 +53,7 @@ "README.md", "deps/grpc/etc/", "index.js", + "index.d.ts", "src/*.js", "ext/*.{cc,h}", "deps/grpc/include/grpc/**/*.h", @@ -66,7 +67,7 @@ "binding.gyp" ], "main": "index.js", - "typings": "src/index.d.ts", + "typings": "index.d.ts", "license": "Apache-2.0", "jshintConfig": { "bitwise": true, diff --git a/packages/grpc-native-core/templates/package.json.template b/packages/grpc-native-core/templates/package.json.template index 4a119737..a4eb88f7 100644 --- a/packages/grpc-native-core/templates/package.json.template +++ b/packages/grpc-native-core/templates/package.json.template @@ -18,7 +18,7 @@ } ], "directories": { - "lib": "src/node/src" + "lib": "src" }, "scripts": { "build": "./node_modules/.bin/node-pre-gyp build", @@ -55,6 +55,7 @@ "README.md", "deps/grpc/etc/", "index.js", + "index.d.ts", "src/*.js", "ext/*.{cc,h}", "deps/grpc/include/grpc/**/*.h", @@ -68,7 +69,7 @@ "binding.gyp" ], "main": "index.js", - "typings": "src/index.d.ts", + "typings": "index.d.ts", "license": "Apache-2.0", "jshintConfig": { "bitwise": true, From 28a7a725b4bbf955a3614aded3a58d486452793f Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Wed, 1 Nov 2017 09:49:40 -0700 Subject: [PATCH 3/7] Update submodule to v1.7.0 and regenerate packages --- packages/grpc-native-core/binding.gyp | 3 +++ packages/grpc-native-core/deps/grpc | 2 +- packages/grpc-native-core/package.json | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/grpc-native-core/binding.gyp b/packages/grpc-native-core/binding.gyp index 9c8646c0..e2c3a743 100644 --- a/packages/grpc-native-core/binding.gyp +++ b/packages/grpc-native-core/binding.gyp @@ -611,6 +611,7 @@ 'deps/grpc/src/core/lib/support/env_linux.c', 'deps/grpc/src/core/lib/support/env_posix.c', 'deps/grpc/src/core/lib/support/env_windows.c', + 'deps/grpc/src/core/lib/support/fork.c', 'deps/grpc/src/core/lib/support/histogram.c', 'deps/grpc/src/core/lib/support/host_port.c', 'deps/grpc/src/core/lib/support/log.c', @@ -693,6 +694,8 @@ 'deps/grpc/src/core/lib/iomgr/ev_windows.c', 'deps/grpc/src/core/lib/iomgr/exec_ctx.c', 'deps/grpc/src/core/lib/iomgr/executor.c', + 'deps/grpc/src/core/lib/iomgr/fork_posix.c', + 'deps/grpc/src/core/lib/iomgr/fork_windows.c', 'deps/grpc/src/core/lib/iomgr/gethostname_fallback.c', 'deps/grpc/src/core/lib/iomgr/gethostname_host_name_max.c', 'deps/grpc/src/core/lib/iomgr/gethostname_sysconf.c', diff --git a/packages/grpc-native-core/deps/grpc b/packages/grpc-native-core/deps/grpc index 6b69f2a1..ed9929e2 160000 --- a/packages/grpc-native-core/deps/grpc +++ b/packages/grpc-native-core/deps/grpc @@ -1 +1 @@ -Subproject commit 6b69f2a164c631fe4d14e799a6e7a20e560ce0aa +Subproject commit ed9929e2465f3a93d81ae45fa48ed2a82d63e831 diff --git a/packages/grpc-native-core/package.json b/packages/grpc-native-core/package.json index 227c4a08..55f5294b 100644 --- a/packages/grpc-native-core/package.json +++ b/packages/grpc-native-core/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.7.0-pre1", + "version": "1.7.0", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "https://grpc.io/", From 6cb28887e6c2ce70d3c618b16e50fe3d24661041 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Thu, 2 Nov 2017 10:13:13 -0700 Subject: [PATCH 4/7] Update to 1.7.1, add Node 9 support, fix generated file permissions --- packages/grpc-native-core/deps/grpc | 2 +- packages/grpc-native-core/package.json | 4 ++-- packages/grpc-native-core/templates/package.json.template | 2 +- .../tools/run_tests/artifacts/build_artifact_node.bat | 2 +- .../tools/run_tests/artifacts/build_artifact_node.sh | 4 +++- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/grpc-native-core/deps/grpc b/packages/grpc-native-core/deps/grpc index ed9929e2..279af911 160000 --- a/packages/grpc-native-core/deps/grpc +++ b/packages/grpc-native-core/deps/grpc @@ -1 +1 @@ -Subproject commit ed9929e2465f3a93d81ae45fa48ed2a82d63e831 +Subproject commit 279af9113481890bcc2c554c5d723c0b3c47f707 diff --git a/packages/grpc-native-core/package.json b/packages/grpc-native-core/package.json index 55f5294b..f3fb5713 100644 --- a/packages/grpc-native-core/package.json +++ b/packages/grpc-native-core/package.json @@ -1,6 +1,6 @@ { "name": "grpc", - "version": "1.7.0", + "version": "1.7.1", "author": "Google Inc.", "description": "gRPC Library for Node", "homepage": "https://grpc.io/", @@ -31,7 +31,7 @@ "arguejs": "^0.2.3", "lodash": "^4.15.0", "nan": "^2.0.0", - "node-pre-gyp": "^0.6.35", + "node-pre-gyp": "^0.6.39", "protobufjs": "^5.0.0" }, "devDependencies": { diff --git a/packages/grpc-native-core/templates/package.json.template b/packages/grpc-native-core/templates/package.json.template index a4eb88f7..d8a5fb74 100644 --- a/packages/grpc-native-core/templates/package.json.template +++ b/packages/grpc-native-core/templates/package.json.template @@ -33,7 +33,7 @@ "arguejs": "^0.2.3", "lodash": "^4.15.0", "nan": "^2.0.0", - "node-pre-gyp": "^0.6.35", + "node-pre-gyp": "^0.6.39", "protobufjs": "^5.0.0" }, "devDependencies": { diff --git a/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.bat b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.bat index 990ed7f4..9563da7d 100644 --- a/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.bat +++ b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.bat @@ -12,7 +12,7 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. -set node_versions=4.0.0 5.0.0 6.0.0 7.0.0 8.0.0 +set node_versions=4.0.0 5.0.0 6.0.0 7.0.0 8.0.0 9.0.0 set electron_versions=1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 1.7.0 diff --git a/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.sh b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.sh index 5669fe29..3b4d0456 100755 --- a/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.sh +++ b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_node.sh @@ -20,6 +20,8 @@ source ~/.nvm/nvm.sh nvm use 8 set -ex +umask 022 + cd $(dirname $0)/../../.. rm -rf build || true @@ -28,7 +30,7 @@ mkdir -p "${ARTIFACTS_OUT}" npm update -node_versions=( 4.0.0 5.0.0 6.0.0 7.0.0 8.0.0 ) +node_versions=( 4.0.0 5.0.0 6.0.0 7.0.0 8.0.0 9.0.0 ) electron_versions=( 1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 1.7.0 ) From 45a3ac256cb5642069b359da72bf1fdbc6e362aa Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Thu, 16 Nov 2017 09:51:32 -0800 Subject: [PATCH 5/7] Remove now-incorrect assertion in completion queue wrapper --- packages/grpc-native-core/ext/completion_queue.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/grpc-native-core/ext/completion_queue.cc b/packages/grpc-native-core/ext/completion_queue.cc index 0cee1a75..7a149532 100644 --- a/packages/grpc-native-core/ext/completion_queue.cc +++ b/packages/grpc-native-core/ext/completion_queue.cc @@ -64,7 +64,6 @@ grpc_completion_queue *GetCompletionQueue() { return queue; } void CompletionQueueNext() { if (pending_batches == 0) { - GPR_ASSERT(!uv_is_active((uv_handle_t *)&prepare)); uv_prepare_start(&prepare, drain_completion_queue); } pending_batches++; From 452abe73b2da7b38e0b889574e72b3bebdd35bfb Mon Sep 17 00:00:00 2001 From: "Bernhard K. Weisshuhn" Date: Mon, 6 Nov 2017 15:44:52 +0100 Subject: [PATCH 6/7] =?UTF-8?q?Build=20different=20binaries=20for=20differ?= =?UTF-8?q?ent=20libc=E2=80=99s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #81 --- packages/grpc-native-core/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/grpc-native-core/package.json b/packages/grpc-native-core/package.json index f3fb5713..687529e5 100644 --- a/packages/grpc-native-core/package.json +++ b/packages/grpc-native-core/package.json @@ -43,10 +43,10 @@ }, "binary": { "module_name": "grpc_node", - "module_path": "src/node/extension_binary/{node_abi}-{platform}-{arch}", + "module_path": "src/node/extension_binary/{node_abi}-{platform}-{arch}-{libc}", "host": "https://storage.googleapis.com/", "remote_path": "grpc-precompiled-binaries/node/{name}/v{version}", - "package_name": "{node_abi}-{platform}-{arch}.tar.gz" + "package_name": "{node_abi}-{platform}-{arch}-{libc}.tar.gz" }, "files": [ "LICENSE", From d4e12d1dbb87a801a828bda34c1132399f0f430d Mon Sep 17 00:00:00 2001 From: "Bernhard K. Weisshuhn" Date: Tue, 7 Nov 2017 08:31:19 +0100 Subject: [PATCH 7/7] Add libc variable to binary paths in package.json template --- packages/grpc-native-core/templates/package.json.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/grpc-native-core/templates/package.json.template b/packages/grpc-native-core/templates/package.json.template index d8a5fb74..f3d471c5 100644 --- a/packages/grpc-native-core/templates/package.json.template +++ b/packages/grpc-native-core/templates/package.json.template @@ -45,10 +45,10 @@ }, "binary": { "module_name": "grpc_node", - "module_path": "src/node/extension_binary/{node_abi}-{platform}-{arch}", + "module_path": "src/node/extension_binary/{node_abi}-{platform}-{arch}-{libc}", "host": "https://storage.googleapis.com/", "remote_path": "grpc-precompiled-binaries/node/{name}/v{version}", - "package_name": "{node_abi}-{platform}-{arch}.tar.gz" + "package_name": "{node_abi}-{platform}-{arch}-{libc}.tar.gz" }, "files": [ "LICENSE",