diff --git a/PACKAGE-COMPARISON.md b/PACKAGE-COMPARISON.md index 8a62c914..c741474b 100644 --- a/PACKAGE-COMPARISON.md +++ b/PACKAGE-COMPARISON.md @@ -3,7 +3,7 @@ Feature | `grpc` | `@grpc/grpc-js` --------|--------|---------- Client | :heavy_check_mark: | :heavy_check_mark: -Server | :heavy_check_mark: | :x: +Server | :heavy_check_mark: | :heavy_check_mark: Unary RPCs | :heavy_check_mark: | :heavy_check_mark: Streaming RPCs | :heavy_check_mark: | :heavy_check_mark: Deadlines | :heavy_check_mark: | :heavy_check_mark: @@ -17,8 +17,8 @@ Connection Keepalives | :heavy_check_mark: | :heavy_check_mark: HTTP Connect Support | :heavy_check_mark: | :x: Retries | :heavy_check_mark: | :x: Stats/tracing/monitoring | :heavy_check_mark: | :x: -Load Balancing | :heavy_check_mark: | :x: -Initial Metadata Options | :heavy_check_mark: | :x: +Load Balancing | :heavy_check_mark: | Pick first and round robin +Initial Metadata Options | :heavy_check_mark: | only `waitForReady` Other Properties | `grpc` | `@grpc/grpc-js` -----------------|--------|---------------- @@ -37,5 +37,9 @@ In addition, all channel arguments defined in [this header file](https://github. - `grpc.keepalive_time_ms` - `grpc.keepalive_timeout_ms` - `grpc.service_config` + - `grpc.max_concurrent_streams` + - `grpc.initial_reconnect_backoff_ms` + - `grpc.max_reconnect_backoff_ms` + - `grpc.use_local_subchannel_pool` - `channelOverride` - `channelFactoryOverride` diff --git a/doc/compression.md b/doc/compression.md new file mode 100644 index 00000000..bfe325a0 --- /dev/null +++ b/doc/compression.md @@ -0,0 +1,31 @@ +# Compression + +## Client side +The preferred method for configuring message compression on a client is to pass `options` when the client object is instantiated. + +These two options control compression behavior: + +**grpc.default_compression_algorithm** (int) + +Default compression algorithm for the channel, applies to sending messages. + +Possible values for this option are: +- `0` - No compression +- `1` - Compress with DEFLATE algorithm +- `2` - Compress with GZIP algorithm +- `3` - Stream compression with GZIP algorithm + +**grpc.default_compression_level** (int) + +Default compression level for the channel, applies to receiving messages. + +Possible values for this option are: +- `0` - None +- `1` - Low level +- `2` - Medium level +- `3` - High level + +### Code example +```javascript +client = new ExampleClient("example.com", credentials.createInsecure(), {'grpc.default_compression_algorithm': 2, 'grpc.default_compression_level': 2}); +``` diff --git a/packages/grpc-js/package.json b/packages/grpc-js/package.json index 4c7030e4..b3109706 100644 --- a/packages/grpc-js/package.json +++ b/packages/grpc-js/package.json @@ -1,6 +1,6 @@ { "name": "@grpc/grpc-js", - "version": "0.6.11", + "version": "0.6.15", "description": "gRPC Library for Node - pure JS implementation", "homepage": "https://grpc.io/", "repository": "https://github.com/grpc/grpc-node/tree/master/packages/grpc-js", @@ -58,7 +58,8 @@ "semver": "^6.2.0" }, "files": [ - "build/src/*.{js,d.ts}", + "src/*.ts", + "build/src/*.{js,d.ts,js.map}", "LICENSE" ] } diff --git a/packages/grpc-js/src/call-stream.ts b/packages/grpc-js/src/call-stream.ts index c7def569..9627ef2c 100644 --- a/packages/grpc-js/src/call-stream.ts +++ b/packages/grpc-js/src/call-stream.ts @@ -25,6 +25,10 @@ import { Metadata } from './metadata'; import { StreamDecoder } from './stream-decoder'; import { ChannelImplementation } from './channel'; import { Subchannel } from './subchannel'; +import * as logging from './logging'; +import { LogVerbosity } from './constants'; + +const TRACER_NAME = 'call_stream'; const { HTTP2_HEADER_STATUS, @@ -202,7 +206,8 @@ export class Http2CallStream implements Call { private readonly channel: ChannelImplementation, private readonly options: CallStreamOptions, filterStackFactory: FilterStackFactory, - private readonly channelCallCredentials: CallCredentials + private readonly channelCallCredentials: CallCredentials, + private readonly callNumber: number ) { this.filterStack = filterStackFactory.createFilter(this); this.credentials = channelCallCredentials; @@ -227,6 +232,14 @@ export class Http2CallStream implements Call { } } + private trace(text: string): void { + logging.trace( + LogVerbosity.DEBUG, + TRACER_NAME, + '[' + this.callNumber + '] ' + text + ); + } + /** * On first call, emits a 'status' event with the given StatusObject. * Subsequent calls are no-ops. @@ -236,6 +249,13 @@ export class Http2CallStream implements Call { /* If the status is OK and a new status comes in (e.g. from a * deserialization failure), that new status takes priority */ if (this.finalStatus === null || this.finalStatus.code === Status.OK) { + this.trace( + 'ended with status: code=' + + status.code + + ' details="' + + status.details + + '"' + ); this.finalStatus = status; this.maybeOutputStatus(); } @@ -259,6 +279,10 @@ export class Http2CallStream implements Call { } private push(message: Buffer): void { + this.trace( + 'pushing to reader message of length ' + + (message instanceof Buffer ? message.length : null) + ); this.canPush = false; process.nextTick(() => { this.listener!.onReceiveMessage(message); @@ -282,6 +306,9 @@ export class Http2CallStream implements Call { this.http2Stream!.pause(); this.push(message); } else { + this.trace( + 'unpushedReadMessages.push message of length ' + message.length + ); this.unpushedReadMessages.push(message); } if (this.unfilteredReadMessages.length > 0) { @@ -299,6 +326,7 @@ export class Http2CallStream implements Call { this.maybeOutputStatus(); return; } + this.trace('filterReceivedMessage of length ' + framedMessage.length); this.isReadFilterPending = true; this.filterStack .receiveMessage(Promise.resolve(framedMessage)) @@ -310,6 +338,12 @@ export class Http2CallStream implements Call { private tryPush(messageBytes: Buffer): void { if (this.isReadFilterPending) { + this.trace( + '[' + + this.callNumber + + '] unfilteredReadMessages.push message of length ' + + (messageBytes && messageBytes.length) + ); this.unfilteredReadMessages.push(messageBytes); } else { this.filterReceivedMessage(messageBytes); @@ -317,6 +351,7 @@ export class Http2CallStream implements Call { } private handleTrailers(headers: http2.IncomingHttpHeaders) { + this.trace('received HTTP/2 trailing headers frame'); const code: Status = this.mappedStatusCode; const details = ''; let metadata: Metadata; @@ -350,11 +385,15 @@ export class Http2CallStream implements Call { if (this.finalStatus !== null) { stream.close(NGHTTP2_CANCEL); } else { + this.trace( + 'attachHttp2Stream from subchannel ' + subchannel.getAddress() + ); this.http2Stream = stream; this.subchannel = subchannel; subchannel.addDisconnectListener(this.disconnectListener); subchannel.callRef(); stream.on('response', (headers, flags) => { + this.trace('received HTTP/2 headers frame'); switch (headers[':status']) { // TODO(murgatroid99): handle 100 and 101 case 400: @@ -408,9 +447,11 @@ export class Http2CallStream implements Call { }); stream.on('trailers', this.handleTrailers.bind(this)); stream.on('data', (data: Buffer) => { + this.trace('receive HTTP/2 data frame of length ' + data.length); const messages = this.decoder.write(data); for (const message of messages) { + this.trace('parsed message of length ' + message.length); this.tryPush(message); } }); @@ -419,6 +460,7 @@ export class Http2CallStream implements Call { this.maybeOutputStatus(); }); stream.on('close', () => { + this.trace('HTTP/2 stream closed with code ' + stream.rstCode); let code: Status; let details = ''; switch (stream.rstCode) { @@ -477,6 +519,7 @@ export class Http2CallStream implements Call { } start(metadata: Metadata, listener: InterceptingListener) { + this.trace('Sending metadata'); this.listener = listener; this.channel._startCallStream(this, metadata); } @@ -553,11 +596,13 @@ export class Http2CallStream implements Call { !this.isWriteFilterPending && this.http2Stream !== null ) { + this.trace('calling end() on HTTP/2 stream'); this.http2Stream.end(); } } sendMessageWithContext(context: MessageContext, message: Buffer) { + this.trace('write() called with message of length ' + message.length); const writeObj: WriteObject = { message, flags: context.flags, @@ -577,6 +622,7 @@ export class Http2CallStream implements Call { } halfClose() { + this.trace('end() called'); this.writesClosed = true; this.maybeCloseWrites(); } diff --git a/packages/grpc-js/src/channel-credentials.ts b/packages/grpc-js/src/channel-credentials.ts index c29885fa..f42c7e93 100644 --- a/packages/grpc-js/src/channel-credentials.ts +++ b/packages/grpc-js/src/channel-credentials.ts @@ -18,7 +18,7 @@ import { ConnectionOptions, createSecureContext, PeerCertificate } from 'tls'; import { CallCredentials } from './call-credentials'; -import { Call } from '.'; +import {CIPHER_SUITES, getDefaultRootsData} from './tls-helpers'; // tslint:disable-next-line:no-any function verifyIsBufferOrNull(obj: any, friendlyName: string): void { @@ -141,7 +141,7 @@ export abstract class ChannelCredentials { ); } return new SecureChannelCredentialsImpl( - rootCerts || null, + rootCerts || getDefaultRootsData(), privateKey || null, certChain || null, verifyOptions || {} @@ -190,6 +190,7 @@ class SecureChannelCredentialsImpl extends ChannelCredentials { ca: rootCerts || undefined, key: privateKey || undefined, cert: certChain || undefined, + ciphers: CIPHER_SUITES }); this.connectionOptions = { secureContext }; if (verifyOptions && verifyOptions.checkServerIdentity) { diff --git a/packages/grpc-js/src/channel-options.ts b/packages/grpc-js/src/channel-options.ts index ad905b9f..9d92b393 100644 --- a/packages/grpc-js/src/channel-options.ts +++ b/packages/grpc-js/src/channel-options.ts @@ -26,6 +26,10 @@ export interface ChannelOptions { 'grpc.keepalive_time_ms'?: number; 'grpc.keepalive_timeout_ms'?: number; 'grpc.service_config'?: string; + 'grpc.max_concurrent_streams'?: number; + 'grpc.initial_reconnect_backoff_ms'?: number; + 'grpc.max_reconnect_backoff_ms'?: number; + 'grpc.use_local_subchannel_pool'?: number; [key: string]: string | number | undefined; } @@ -41,6 +45,10 @@ export const recognizedOptions = { 'grpc.keepalive_time_ms': true, 'grpc.keepalive_timeout_ms': true, 'grpc.service_config': true, + 'grpc.max_concurrent_streams': true, + 'grpc.initial_reconnect_backoff_ms': true, + 'grpc.max_reconnect_backoff_ms': true, + 'grpc.use_local_subchannel_pool': true, }; export function channelOptionsEqual( diff --git a/packages/grpc-js/src/channel.ts b/packages/grpc-js/src/channel.ts index c1ab8459..6cf987cb 100644 --- a/packages/grpc-js/src/channel.ts +++ b/packages/grpc-js/src/channel.ts @@ -47,6 +47,17 @@ export enum ConnectivityState { SHUTDOWN, } +let nextCallNumber = 0; + +function getNewCallNumber(): number { + const callNumber = nextCallNumber; + nextCallNumber += 1; + if (nextCallNumber >= Number.MAX_SAFE_INTEGER) { + nextCallNumber = 0; + } + return callNumber; +} + /** * An interface that represents a communication channel to a server specified * by a given address. @@ -129,8 +140,9 @@ export class ChannelImplementation implements Channel { private readonly credentials: ChannelCredentials, private readonly options: ChannelOptions ) { - // TODO(murgatroid99): check channel arg for getting a private pool - this.subchannelPool = getSubchannelPool(true); + /* The global boolean parameter to getSubchannelPool has the inverse meaning to what + * the grpc.use_local_subchannel_pool channel option means. */ + this.subchannelPool = getSubchannelPool((options['grpc.use_local_subchannel_pool'] ?? 0) === 0); const channelControlHelper: ChannelControlHelper = { createSubchannel: ( subchannelAddress: string, @@ -216,10 +228,17 @@ export class ChannelImplementation implements Channel { pickResult.subchannel!.getConnectivityState() === ConnectivityState.READY ) { - pickResult.subchannel!.startCallStream( - finalMetadata, - callStream - ); + try { + pickResult.subchannel!.startCallStream( + finalMetadata, + callStream + ); + } catch (error) { + callStream.cancelWithStatus( + Status.UNAVAILABLE, + 'Failed to start call on picked subchannel' + ); + } } else { callStream.cancelWithStatus( Status.UNAVAILABLE, @@ -304,8 +323,12 @@ export class ChannelImplementation implements Channel { return this.target; } - getConnectivityState() { - return this.connectivityState; + getConnectivityState(tryToConnect: boolean) { + const connectivityState = this.connectivityState; + if (tryToConnect) { + this.resolvingLoadBalancer.exitIdle(); + } + return connectivityState; } watchConnectivityState( @@ -346,6 +369,18 @@ export class ChannelImplementation implements Channel { if (this.connectivityState === ConnectivityState.SHUTDOWN) { throw new Error('Channel has been shut down'); } + const callNumber = getNewCallNumber(); + trace( + LogVerbosity.DEBUG, + 'channel', + this.target + + ' createCall [' + + callNumber + + '] method="' + + method + + '", deadline=' + + deadline + ); const finalOptions: CallStreamOptions = { deadline: deadline === null || deadline === undefined ? Infinity : deadline, @@ -358,7 +393,8 @@ export class ChannelImplementation implements Channel { this, finalOptions, this.filterStackFactory, - this.credentials._getCallCredentials() + this.credentials._getCallCredentials(), + callNumber ); return stream; } diff --git a/packages/grpc-js/src/index.ts b/packages/grpc-js/src/index.ts index e38c8c96..a80b08b9 100644 --- a/packages/grpc-js/src/index.ts +++ b/packages/grpc-js/src/index.ts @@ -85,11 +85,11 @@ export interface OAuth2Client { callback: ( err: Error | null, headers?: { - Authorization: string; + [index: string]: string; } ) => void ) => void; - getRequestHeaders: (url?: string) => Promise<{ Authorization: string }>; + getRequestHeaders: (url?: string) => Promise<{ [index: string]: string }>; } /**** Client Credentials ****/ @@ -109,7 +109,7 @@ export const credentials = mixin( (options, callback) => { // google-auth-library pre-v2.0.0 does not have getRequestHeaders // but has getRequestMetadata, which is deprecated in v2.0.0 - let getHeaders: Promise<{ Authorization: string }>; + let getHeaders: Promise<{ [index: string]: string }>; if (typeof googleCredentials.getRequestHeaders === 'function') { getHeaders = googleCredentials.getRequestHeaders( options.service_url @@ -131,7 +131,9 @@ export const credentials = mixin( getHeaders.then( headers => { const metadata = new Metadata(); - metadata.add('authorization', headers.Authorization); + for (const key of Object.keys(headers)) { + metadata.add(key, headers[key]); + } callback(null, metadata); }, err => { diff --git a/packages/grpc-js/src/metadata.ts b/packages/grpc-js/src/metadata.ts index 67a23960..4b41aa25 100644 --- a/packages/grpc-js/src/metadata.ts +++ b/packages/grpc-js/src/metadata.ts @@ -16,6 +16,8 @@ */ import * as http2 from 'http2'; +import { log } from './logging'; +import { LogVerbosity } from './constants'; const LEGAL_KEY_REGEX = /^[0-9a-z_.-]+$/; const LEGAL_NON_BINARY_VALUE_REGEX = /^[ -~]*$/; @@ -34,6 +36,10 @@ function isBinaryKey(key: string): boolean { return key.endsWith('-bin'); } +function isCustomMetadata(key: string): boolean { + return !key.startsWith('grpc-'); +} + function normalizeKey(key: string): string { return key.toLowerCase(); } @@ -258,9 +264,13 @@ export class Metadata { result.add(key, Buffer.from(value, 'base64')); }); } else if (values !== undefined) { - values.split(',').forEach(v => { - result.add(key, Buffer.from(v.trim(), 'base64')); - }); + if (isCustomMetadata(key)) { + values.split(',').forEach(v => { + result.add(key, Buffer.from(v.trim(), 'base64')); + }); + } else { + result.add(key, Buffer.from(values, 'base64')); + } } } else { if (Array.isArray(values)) { @@ -268,12 +278,16 @@ export class Metadata { result.add(key, value); }); } else if (values !== undefined) { - values.split(',').forEach(v => result.add(key, v.trim())); + if (isCustomMetadata(key)) { + values.split(',').forEach(v => result.add(key, v.trim())); + } else { + result.add(key, values); + } } } } catch (error) { - error.message = `Failed to add metadata entry ${key}: ${values}. ${error.message}`; - process.emitWarning(error); + const message = `Failed to add metadata entry ${key}: ${values}. ${error.message}. For more information see https://github.com/grpc/grpc-node/issues/1173`; + log(LogVerbosity.ERROR, message); } }); return result; diff --git a/packages/grpc-js/src/resolver-dns.ts b/packages/grpc-js/src/resolver-dns.ts index e98bf66b..9f91d70a 100644 --- a/packages/grpc-js/src/resolver-dns.ts +++ b/packages/grpc-js/src/resolver-dns.ts @@ -115,14 +115,15 @@ const dnsLookupPromise = util.promisify(dns.lookup); function parseIP(target: string): string[] | null { /* These three regular expressions are all mutually exclusive, so we just * want the first one that matches the target string, if any do. */ + const ipv4Match = IPV4_REGEX.exec(target); const match = - IPV4_REGEX.exec(target) || - IPV6_REGEX.exec(target) || - IPV6_BRACKET_REGEX.exec(target); + ipv4Match || IPV6_REGEX.exec(target) || IPV6_BRACKET_REGEX.exec(target); if (match === null) { return null; } - const addr = match[1]; + + // ipv6 addresses should be bracketed + const addr = ipv4Match ? match[1] : `[${match[1]}]`; let port: string; if (match[2]) { port = match[2]; @@ -140,7 +141,11 @@ function mergeArrays(...arrays: T[][]): T[] { const result: T[] = []; for ( let i = 0; - i < Math.max.apply(null, arrays.map(array => array.length)); + i < + Math.max.apply( + null, + arrays.map(array => array.length) + ); i++ ) { for (const array of arrays) { diff --git a/packages/grpc-js/src/resolver.ts b/packages/grpc-js/src/resolver.ts index 512f3c55..3af5da9c 100644 --- a/packages/grpc-js/src/resolver.ts +++ b/packages/grpc-js/src/resolver.ts @@ -125,7 +125,7 @@ export function createResolver( * @param target */ export function getDefaultAuthority(target: string): string { - for (const prefix of Object.keys(registerDefaultResolver)) { + for (const prefix of Object.keys(registeredResolvers)) { if (target.startsWith(prefix)) { return registeredResolvers[prefix].getDefaultAuthority(target); } diff --git a/packages/grpc-js/src/server-credentials.ts b/packages/grpc-js/src/server-credentials.ts index 1fe5f55d..b56cb68a 100644 --- a/packages/grpc-js/src/server-credentials.ts +++ b/packages/grpc-js/src/server-credentials.ts @@ -16,6 +16,7 @@ */ import { SecureServerOptions } from 'http2'; +import {CIPHER_SUITES, getDefaultRootsData} from './tls-helpers'; export interface KeyCertPair { private_key: Buffer; @@ -70,10 +71,11 @@ export abstract class ServerCredentials { } return new SecureServerCredentials({ - ca: rootCerts || undefined, + ca: rootCerts || getDefaultRootsData() || undefined, cert, key, requestCert: checkClientCertificate, + ciphers: CIPHER_SUITES }); } } diff --git a/packages/grpc-js/src/server.ts b/packages/grpc-js/src/server.ts index 90aeea20..733b0309 100644 --- a/packages/grpc-js/src/server.ts +++ b/packages/grpc-js/src/server.ts @@ -45,6 +45,7 @@ import { ServerStatusResponse, } from './server-call'; import { ServerCredentials } from './server-credentials'; +import { ChannelOptions } from './channel-options'; function noop(): void {} @@ -95,8 +96,11 @@ export class Server { >(); private sessions = new Set(); private started = false; + private options: ChannelOptions; - constructor(options?: object) {} + constructor(options?: ChannelOptions) { + this.options = options ?? {}; + } addProtoService(): void { throw new Error('Not implemented. Use addService() instead'); @@ -197,13 +201,16 @@ export class Server { const url = new URL(`http://${port}`); const options: ListenOptions = { host: url.hostname, port: +url.port }; + const serverOptions: http2.ServerOptions = {}; + if ('grpc.max_concurrent_streams' in this.options) { + serverOptions.settings = {maxConcurrentStreams: this.options['grpc.max_concurrent_streams']}; + } if (creds._isSecure()) { - this.http2Server = http2.createSecureServer( - creds._getSettings() as http2.SecureServerOptions - ); + const secureServerOptions = Object.assign(serverOptions, creds._getSettings()!); + this.http2Server = http2.createSecureServer(secureServerOptions); } else { - this.http2Server = http2.createServer(); + this.http2Server = http2.createServer(serverOptions); } this.http2Server.setTimeout(0, noop); diff --git a/packages/grpc-js/src/subchannel.ts b/packages/grpc-js/src/subchannel.ts index eae178e7..b6b282d5 100644 --- a/packages/grpc-js/src/subchannel.ts +++ b/packages/grpc-js/src/subchannel.ts @@ -22,7 +22,7 @@ import { Http2CallStream } from './call-stream'; import { ChannelOptions } from './channel-options'; import { PeerCertificate, checkServerIdentity } from 'tls'; import { ConnectivityState } from './channel'; -import { BackoffTimeout } from './backoff-timeout'; +import { BackoffTimeout, BackoffOptions } from './backoff-timeout'; import { getDefaultAuthority } from './resolver'; import * as logging from './logging'; import { LogVerbosity } from './constants'; @@ -170,6 +170,10 @@ export class Subchannel { clearTimeout(this.keepaliveIntervalId); this.keepaliveTimeoutId = setTimeout(() => {}, 0); clearTimeout(this.keepaliveTimeoutId); + const backoffOptions: BackoffOptions = { + initialDelay: options['grpc.initial_reconnect_backoff_ms'], + maxDelay: options['grpc.max_reconnect_backoff_ms'] + }; this.backoffTimeout = new BackoffTimeout(() => { if (this.continueConnecting) { this.transitionToState( @@ -182,7 +186,7 @@ export class Subchannel { ConnectivityState.IDLE ); } - }); + }, backoffOptions); } /** @@ -395,6 +399,13 @@ export class Subchannel { } callRef() { + trace( + this.subchannelAddress + + ' callRefcount ' + + this.callRefcount + + ' -> ' + + (this.callRefcount + 1) + ); if (this.callRefcount === 0) { if (this.session) { this.session.ref(); @@ -405,6 +416,13 @@ export class Subchannel { } callUnref() { + trace( + this.subchannelAddress + + ' callRefcount ' + + this.callRefcount + + ' -> ' + + (this.callRefcount - 1) + ); this.callRefcount -= 1; if (this.callRefcount === 0) { if (this.session) { @@ -416,10 +434,24 @@ export class Subchannel { } ref() { + trace( + this.subchannelAddress + + ' callRefcount ' + + this.refcount + + ' -> ' + + (this.refcount + 1) + ); this.refcount += 1; } unref() { + trace( + this.subchannelAddress + + ' callRefcount ' + + this.refcount + + ' -> ' + + (this.refcount - 1) + ); this.refcount -= 1; this.checkBothRefcounts(); } diff --git a/packages/grpc-js/src/tls-helpers.ts b/packages/grpc-js/src/tls-helpers.ts new file mode 100644 index 00000000..161666ed --- /dev/null +++ b/packages/grpc-js/src/tls-helpers.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import * as fs from 'fs'; + +export const CIPHER_SUITES: string | undefined = process.env.GRPC_SSL_CIPHER_SUITES; + +const DEFAULT_ROOTS_FILE_PATH = process.env.GRPC_DEFAULT_SSL_ROOTS_FILE_PATH; + +let defaultRootsData: Buffer | null = null; + +export function getDefaultRootsData(): Buffer | null { + if (DEFAULT_ROOTS_FILE_PATH) { + if (defaultRootsData === null) { + defaultRootsData = fs.readFileSync(DEFAULT_ROOTS_FILE_PATH); + } + return defaultRootsData; + } + return null; +} \ No newline at end of file diff --git a/packages/grpc-js/test/test-resolver.ts b/packages/grpc-js/test/test-resolver.ts index a5a6a77e..951291a6 100644 --- a/packages/grpc-js/test/test-resolver.ts +++ b/packages/grpc-js/test/test-resolver.ts @@ -67,6 +67,63 @@ describe('Name Resolver', () => { const resolver = resolverManager.createResolver(target, listener); resolver.updateResolution(); }); + it('Should correctly represent an ipv4 address', done => { + const target = '1.2.3.4'; + const listener: resolverManager.ResolverListener = { + onSuccessfulResolution: ( + addressList: string[], + serviceConfig: ServiceConfig | null, + serviceConfigError: StatusObject | null + ) => { + assert(addressList.includes('1.2.3.4:443')); + // We would check for the IPv6 address but it needs to be omitted on some Node versions + done(); + }, + onError: (error: StatusObject) => { + done(new Error(`Failed with status ${error.details}`)); + }, + }; + const resolver = resolverManager.createResolver(target, listener); + resolver.updateResolution(); + }); + it('Should correctly represent an ipv6 address', done => { + const target = '::1'; + const listener: resolverManager.ResolverListener = { + onSuccessfulResolution: ( + addressList: string[], + serviceConfig: ServiceConfig | null, + serviceConfigError: StatusObject | null + ) => { + assert(addressList.includes('[::1]:443')); + // We would check for the IPv6 address but it needs to be omitted on some Node versions + done(); + }, + onError: (error: StatusObject) => { + done(new Error(`Failed with status ${error.details}`)); + }, + }; + const resolver = resolverManager.createResolver(target, listener); + resolver.updateResolution(); + }); + it('Should correctly represent a bracketed ipv6 address', done => { + const target = '[::1]:50051'; + const listener: resolverManager.ResolverListener = { + onSuccessfulResolution: ( + addressList: string[], + serviceConfig: ServiceConfig | null, + serviceConfigError: StatusObject | null + ) => { + assert(addressList.includes('[::1]:50051')); + // We would check for the IPv6 address but it needs to be omitted on some Node versions + done(); + }, + onError: (error: StatusObject) => { + done(new Error(`Failed with status ${error.details}`)); + }, + }; + const resolver = resolverManager.createResolver(target, listener); + resolver.updateResolution(); + }); it('Should resolve a public address', done => { const target = 'example.com'; const listener: resolverManager.ResolverListener = { @@ -194,4 +251,23 @@ describe('Name Resolver', () => { resolver.updateResolution(); }); }); + describe('getDefaultAuthority', () => { + class OtherResolver implements resolverManager.Resolver { + updateResolution() { + return []; + } + + static getDefaultAuthority(target: string): string { + return 'other'; + } + } + + it('Should return the correct authority if a different resolver has been registered', () => { + const target = 'other://name'; + resolverManager.registerResolver('other:', OtherResolver); + + const authority = resolverManager.getDefaultAuthority(target); + assert.equal(authority, 'other'); + }); + }); }); diff --git a/packages/grpc-js/test/test-server-credentials.ts b/packages/grpc-js/test/test-server-credentials.ts index 6c8bd877..ec1740f7 100644 --- a/packages/grpc-js/test/test-server-credentials.ts +++ b/packages/grpc-js/test/test-server-credentials.ts @@ -41,24 +41,16 @@ describe('Server Credentials', () => { const creds = ServerCredentials.createSsl(ca, []); assert.strictEqual(creds._isSecure(), true); - assert.deepStrictEqual(creds._getSettings(), { - ca, - cert: [], - key: [], - requestCert: false, - }); + assert.strictEqual(creds._getSettings()?.ca, ca); }); it('accepts a boolean as the third argument', () => { const creds = ServerCredentials.createSsl(ca, [], true); assert.strictEqual(creds._isSecure(), true); - assert.deepStrictEqual(creds._getSettings(), { - ca, - cert: [], - key: [], - requestCert: true, - }); + const settings = creds._getSettings(); + assert.strictEqual(settings?.ca, ca); + assert.strictEqual(settings?.requestCert, true); }); it('accepts an object with two buffers in the second argument', () => { @@ -66,12 +58,9 @@ describe('Server Credentials', () => { const creds = ServerCredentials.createSsl(null, keyCertPairs); assert.strictEqual(creds._isSecure(), true); - assert.deepStrictEqual(creds._getSettings(), { - ca: undefined, - cert: [cert], - key: [key], - requestCert: false, - }); + const settings = creds._getSettings(); + assert.deepStrictEqual(settings?.cert, [cert]); + assert.deepStrictEqual(settings?.key, [key]); }); it('accepts multiple objects in the second argument', () => { @@ -82,12 +71,9 @@ describe('Server Credentials', () => { const creds = ServerCredentials.createSsl(null, keyCertPairs, false); assert.strictEqual(creds._isSecure(), true); - assert.deepStrictEqual(creds._getSettings(), { - ca: undefined, - cert: [cert, cert], - key: [key, key], - requestCert: false, - }); + const settings = creds._getSettings(); + assert.deepStrictEqual(settings?.cert, [cert, cert]); + assert.deepStrictEqual(settings?.key, [key, key]); }); it('fails if the second argument is not an Array', () => { diff --git a/packages/grpc-js/test/test-server-errors.ts b/packages/grpc-js/test/test-server-errors.ts index ffabaac5..7c611b9b 100644 --- a/packages/grpc-js/test/test-server-errors.ts +++ b/packages/grpc-js/test/test-server-errors.ts @@ -699,6 +699,18 @@ describe('Other conditions', () => { } ); }); + + it('for an error message with a comma', done => { + client.unary( + { error: true, message: 'an error message, with a comma' }, + (err: ServiceError, data: any) => { + assert(err); + assert.strictEqual(err.code, grpc.status.UNKNOWN); + assert.strictEqual(err.details, 'an error message, with a comma'); + done(); + } + ); + }); }); }); diff --git a/packages/grpc-native-core/src/credentials.js b/packages/grpc-native-core/src/credentials.js index 9542acfd..21523f0b 100644 --- a/packages/grpc-native-core/src/credentials.js +++ b/packages/grpc-native-core/src/credentials.js @@ -182,25 +182,25 @@ exports.createFromMetadataGenerator = function(metadata_generator) { }); }; -function getAuthorizationHeaderFromGoogleCredential(google_credential, url, callback) { +function getHeadersFromGoogleCredential(google_credential, url, callback) { // google-auth-library pre-v2.0.0 does not have getRequestHeaders // but has getRequestMetadata, which is deprecated in v2.0.0 if (typeof google_credential.getRequestHeaders === 'function') { google_credential.getRequestHeaders(url) - .then(function(header) { - callback(null, header.Authorization); + .then(function(headers) { + callback(null, headers); }) .catch(function(err) { callback(err); return; }); } else { - google_credential.getRequestMetadata(url, function(err, header) { + google_credential.getRequestMetadata(url, function(err, headers) { if (err) { callback(err); return; } - callback(null, header.Authorization); + callback(null, headers); }); } } @@ -216,15 +216,17 @@ function getAuthorizationHeaderFromGoogleCredential(google_credential, url, call exports.createFromGoogleCredential = function(google_credential) { return exports.createFromMetadataGenerator(function(auth_context, callback) { var service_url = auth_context.service_url; - getAuthorizationHeaderFromGoogleCredential(google_credential, service_url, - function(err, authHeader) { + getHeadersFromGoogleCredential(google_credential, service_url, + function(err, headers) { if (err) { common.log(constants.logVerbosity.INFO, 'Auth error:' + err); callback(err); return; } var metadata = new Metadata(); - metadata.add('authorization', authHeader); + for (const key of Object.keys(headers)) { + metadata.add(key, headers[key]); + } callback(null, metadata); }); }); diff --git a/packages/grpc-native-core/src/metadata.js b/packages/grpc-native-core/src/metadata.js index 279dee18..c5b988fb 100644 --- a/packages/grpc-native-core/src/metadata.js +++ b/packages/grpc-native-core/src/metadata.js @@ -22,6 +22,9 @@ var clone = require('lodash.clone'); var grpc = require('./grpc_extension'); +const common = require('./common'); +const logVerbosity = require('./constants').logVerbosity; + const IDEMPOTENT_REQUEST_FLAG = 0x10; const WAIT_FOR_READY_FLAG = 0x20; const CACHEABLE_REQUEST_FLAG = 0x40; @@ -231,6 +234,12 @@ Metadata._fromCoreRepresentation = function(metadata) { if (metadata) { Object.keys(metadata.metadata).forEach(key => { const value = metadata.metadata[key]; + if (!grpc.metadataKeyIsLegal(key)) { + common.log(logVerbosity.ERROR, + "Warning: possibly corrupted metadata key received: " + + key + ": " + value + + ". Please report this at https://github.com/grpc/grpc-node/issues/1173."); + } newMetadata._internal_repr[key] = clone(value); }); } diff --git a/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_electron.bat b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_electron.bat index 208e056a..56fa530b 100644 --- a/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_electron.bat +++ b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_electron.bat @@ -14,7 +14,7 @@ set arch_list=ia32 x64 -set electron_versions=1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 1.7.0 1.8.0 2.0.0 3.0.0 3.1.0 4.1.0 4.2.0 5.0.0 6.0.0 7.0.0 +set electron_versions=1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 1.7.0 1.8.0 2.0.0 3.0.0 3.1.0 4.1.0 4.2.0 5.0.0 6.0.0 6.1.0 7.0.0 7.1.0 set PATH=%PATH%;C:\Program Files\nodejs\;%APPDATA%\npm diff --git a/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_electron.sh b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_electron.sh index bc288410..f6d3c962 100755 --- a/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_electron.sh +++ b/packages/grpc-native-core/tools/run_tests/artifacts/build_artifact_electron.sh @@ -16,7 +16,7 @@ set -ex arch_list=( ia32 x64 ) -electron_versions=( 1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 1.7.0 1.8.0 2.0.0 3.0.0 3.1.0 4.1.0 4.2.0 5.0.0 6.0.0 7.0.0 ) +electron_versions=( 1.0.0 1.1.0 1.2.0 1.3.0 1.4.0 1.5.0 1.6.0 1.7.0 1.8.0 2.0.0 3.0.0 3.1.0 4.1.0 4.2.0 5.0.0 6.0.0 6.1.0 7.0.0 7.1.0 ) umask 022 diff --git a/packages/grpc-tools/package.json b/packages/grpc-tools/package.json index 8f4947f3..2bf3feff 100644 --- a/packages/grpc-tools/package.json +++ b/packages/grpc-tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.8.0", + "version": "1.8.1", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "https://grpc.io/", diff --git a/packages/grpc-tools/src/node_generator.cc b/packages/grpc-tools/src/node_generator.cc index 72059470..77a5abfd 100644 --- a/packages/grpc-tools/src/node_generator.cc +++ b/packages/grpc-tools/src/node_generator.cc @@ -181,7 +181,7 @@ void PrintMethod(const MethodDescriptor* method, Printer* out) { void PrintService(const ServiceDescriptor* service, Printer* out, const Parameters& params) { map template_vars; - out->Print(GetNodeComments(service, true).c_str()); + out->PrintRaw(GetNodeComments(service, true).c_str()); template_vars["name"] = service->name(); template_vars["full_name"] = service->full_name(); if (params.generate_package_definition) { @@ -193,11 +193,11 @@ void PrintService(const ServiceDescriptor* service, Printer* out, for (int i = 0; i < service->method_count(); i++) { grpc::string method_name = grpc_generator::LowercaseFirstLetter(service->method(i)->name()); - out->Print(GetNodeComments(service->method(i), true).c_str()); + out->PrintRaw(GetNodeComments(service->method(i), true).c_str()); out->Print("$method_name$: ", "method_name", method_name); PrintMethod(service->method(i), out); out->Print(",\n"); - out->Print(GetNodeComments(service->method(i), false).c_str()); + out->PrintRaw(GetNodeComments(service->method(i), false).c_str()); } out->Outdent(); out->Print("};\n\n"); @@ -206,7 +206,7 @@ void PrintService(const ServiceDescriptor* service, Printer* out, "exports.$name$Client = " "grpc.makeGenericClientConstructor($name$Service);\n"); } - out->Print(GetNodeComments(service, false).c_str()); + out->PrintRaw(GetNodeComments(service, false).c_str()); } void PrintImports(const FileDescriptor* file, Printer* out, @@ -276,7 +276,7 @@ grpc::string GenerateFile(const FileDescriptor* file, PrintServices(file, &out, params); - out.Print(GetNodeComments(file, false).c_str()); + out.PrintRaw(GetNodeComments(file, false).c_str()); } return output; } diff --git a/packages/grpc-tools/src/node_plugin.cc b/packages/grpc-tools/src/node_plugin.cc index a502dca8..4ed11081 100644 --- a/packages/grpc-tools/src/node_plugin.cc +++ b/packages/grpc-tools/src/node_plugin.cc @@ -43,7 +43,6 @@ class NodeGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { grpc_generator::tokenize(parameter, ","); for (auto parameter_string = parameters_list.begin(); parameter_string != parameters_list.end(); parameter_string++) { - printf("%s", parameter_string); if (*parameter_string == "generate_package_definition") { generator_parameters.generate_package_definition = true; } diff --git a/test/api/error_test.js b/test/api/error_test.js index 23a8d0d8..ea3bec91 100644 --- a/test/api/error_test.js +++ b/test/api/error_test.js @@ -553,6 +553,14 @@ describe(`${anyGrpc.clientName} client -> ${anyGrpc.serverName} server`, functio done(); }); }); + it('for an error message with a comma', function(done) { + client.unary({error: true, message: 'a message, with a comma'}, function(err, data) { + assert(err); + assert.strictEqual(err.code, clientGrpc.status.UNKNOWN); + assert.strictEqual(err.details, 'a message, with a comma'); + done(); + }); + }); }); }); }); \ No newline at end of file diff --git a/test/kokoro.bat b/test/kokoro.bat index 1f0fabfc..3190c141 100644 --- a/test/kokoro.bat +++ b/test/kokoro.bat @@ -17,7 +17,6 @@ cd /d %~dp0 cd .. -git submodule update --init -git submodule foreach --recursive git submodule update --init +git submodule update --init --recursive .\run-tests.bat diff --git a/test/kokoro.sh b/test/kokoro.sh index c4b9373b..667db112 100755 --- a/test/kokoro.sh +++ b/test/kokoro.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2017 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Deleting Ruby. +rm -rf ~/.rvm + set -e cd $(dirname $0)/.. # Install gRPC and its submodules. -git submodule update --init -git submodule foreach --recursive git submodule update --init +git submodule update --init --recursive ./packages/grpc-native-core/tools/buildgen/generate_projects.sh diff --git a/tools/release/kokoro-electron.bat b/tools/release/kokoro-electron.bat index c27db480..dde9b134 100644 --- a/tools/release/kokoro-electron.bat +++ b/tools/release/kokoro-electron.bat @@ -27,8 +27,7 @@ call npm install -g node-gyp@3 cd /d %~dp0 cd ..\.. -git submodule update --init -git submodule foreach --recursive git submodule update --init +git submodule update --init --recursive set ARTIFACTS_OUT=artifacts cd packages\grpc-native-core diff --git a/tools/release/kokoro-electron.sh b/tools/release/kokoro-electron.sh index 1305c1ae..fd833503 100755 --- a/tools/release/kokoro-electron.sh +++ b/tools/release/kokoro-electron.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2018 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Deleting Ruby. +rm -rf ~/.rvm + curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" @@ -28,8 +31,7 @@ cd $(dirname $0)/../.. base_dir=$(pwd) # Install gRPC and its submodules. -git submodule update --init -git submodule foreach --recursive git submodule update --init +git submodule update --init --recursive pip install mako ./packages/grpc-native-core/tools/buildgen/generate_projects.sh diff --git a/tools/release/kokoro-grpc-tools.sh b/tools/release/kokoro-grpc-tools.sh index b672c0cb..267d6cc2 100755 --- a/tools/release/kokoro-grpc-tools.sh +++ b/tools/release/kokoro-grpc-tools.sh @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Deleting Ruby. +rm -rf ~/.rvm + set -e cd $(dirname $0)/../.. base_dir=$(pwd) diff --git a/tools/release/kokoro-nodejs.bat b/tools/release/kokoro-nodejs.bat index faa6364a..dd6aecfc 100644 --- a/tools/release/kokoro-nodejs.bat +++ b/tools/release/kokoro-nodejs.bat @@ -27,8 +27,7 @@ call npm install -g node-gyp@3 cd /d %~dp0 cd ..\.. -git submodule update --init -git submodule foreach --recursive git submodule update --init +git submodule update --init --recursive set ARTIFACTS_OUT=%cd%\artifacts cd packages\grpc-native-core diff --git a/tools/release/kokoro-nodejs.sh b/tools/release/kokoro-nodejs.sh index c1f90562..7ce0a0b6 100755 --- a/tools/release/kokoro-nodejs.sh +++ b/tools/release/kokoro-nodejs.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2018 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Deleting Ruby. +rm -rf ~/.rvm + curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash export NVM_DIR="$HOME/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" @@ -28,8 +31,7 @@ cd $(dirname $0)/../.. base_dir=$(pwd) # Install gRPC and its submodules. -git submodule update --init -git submodule foreach --recursive git submodule update --init +git submodule update --init --recursive pip install mako ./packages/grpc-native-core/tools/buildgen/generate_projects.sh