From 8bacb12d23506837e75ce21793cc667f286fe4f8 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Fri, 13 Nov 2020 14:54:38 -0800 Subject: [PATCH 01/46] grpc-js-xds: Reset LRS backoff on data, not metadata --- packages/grpc-js-xds/src/xds-client.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/grpc-js-xds/src/xds-client.ts b/packages/grpc-js-xds/src/xds-client.ts index 525db769..da75dc7c 100644 --- a/packages/grpc-js-xds/src/xds-client.ts +++ b/packages/grpc-js-xds/src/xds-client.ts @@ -1018,13 +1018,11 @@ export class XdsClient { this.lrsBackoff.runOnce(); this.lrsCall = this.lrsClient.streamLoadStats(); - this.lrsCall.on('metadata', () => { + this.lrsCall.on('data', (message: LoadStatsResponse__Output) => { /* Once we get any response from the server, we assume that the stream is * in a good state, so we can reset the backoff timer. */ this.lrsBackoff.stop(); this.lrsBackoff.reset(); - }); - this.lrsCall.on('data', (message: LoadStatsResponse__Output) => { if ( message.load_reporting_interval?.seconds !== this.latestLrsSettings?.load_reporting_interval?.seconds || From c050f97534b3332fc0a4818e20415cdffaaaf60e Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Wed, 18 Nov 2020 13:08:06 -0800 Subject: [PATCH 02/46] grpc-js: Make calls use the min of parent and own deadline when both are provided --- packages/grpc-js/package.json | 2 +- packages/grpc-js/src/call-stream.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/grpc-js/package.json b/packages/grpc-js/package.json index 0e445377..22142dcd 100644 --- a/packages/grpc-js/package.json +++ b/packages/grpc-js/package.json @@ -1,6 +1,6 @@ { "name": "@grpc/grpc-js", - "version": "1.2.0", + "version": "1.2.1", "description": "gRPC Library for Node - pure JS implementation", "homepage": "https://grpc.io/", "repository": "https://github.com/grpc/grpc-node/tree/master/packages/grpc-js", diff --git a/packages/grpc-js/src/call-stream.ts b/packages/grpc-js/src/call-stream.ts index 32b85165..91e24ea5 100644 --- a/packages/grpc-js/src/call-stream.ts +++ b/packages/grpc-js/src/call-stream.ts @@ -630,7 +630,11 @@ export class Http2CallStream implements Call { getDeadline(): Deadline { if (this.options.parentCall && this.options.flags & Propagate.DEADLINE) { - return this.options.parentCall.getDeadline(); + const parentDeadline = this.options.parentCall.getDeadline(); + const selfDeadline = this.options.deadline; + const parentDeadlineMsecs = parentDeadline instanceof Date ? parentDeadline.getTime() : parentDeadline; + const selfDeadlineMsecs = selfDeadline instanceof Date ? selfDeadline.getTime() : selfDeadline; + return Math.min(parentDeadlineMsecs, selfDeadlineMsecs); } else { return this.options.deadline; } From d3a1ba6cbf6ddf3c179b44ed98ed8c1d09067fd2 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Wed, 25 Nov 2020 10:04:05 -0800 Subject: [PATCH 03/46] Make grpc-js and grpc-js-xds versions match --- packages/grpc-js-xds/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/grpc-js-xds/package.json b/packages/grpc-js-xds/package.json index 78d90c1d..3008d3ec 100644 --- a/packages/grpc-js-xds/package.json +++ b/packages/grpc-js-xds/package.json @@ -1,6 +1,6 @@ { "name": "@grpc/grpc-js-xds", - "version": "1.0.0", + "version": "1.2.0", "description": "Plugin for @grpc/grpc-js. Adds the xds:// URL scheme and associated features.", "main": "build/src/index.js", "scripts": { @@ -32,13 +32,13 @@ "homepage": "https://github.com/grpc/grpc-node#readme", "devDependencies": { "@grpc/grpc-js": "file:../grpc-js", - "gts": "^2.0.2", - "typescript": "^3.8.3", "@types/gulp": "^4.0.6", "@types/gulp-mocha": "0.0.32", "@types/mocha": "^5.2.6", "@types/node": "^13.11.1", "@types/yargs": "^15.0.5", + "gts": "^2.0.2", + "typescript": "^3.8.3", "yargs": "^15.4.1" }, "dependencies": { From a006be07f452da7e69fb3c5779e61c6092f150b8 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Wed, 25 Nov 2020 12:33:36 -0800 Subject: [PATCH 04/46] grpc-js-xds: Shutdown the xDS client used by the resolver when the channel shuts down --- packages/grpc-js-xds/package.json | 2 +- packages/grpc-js-xds/src/resolver-xds.ts | 13 ++++++++----- packages/grpc-js/package.json | 2 +- packages/grpc-js/src/resolver-dns.ts | 6 ++++++ packages/grpc-js/src/resolver-uds.ts | 4 ++++ packages/grpc-js/src/resolver.ts | 5 +++++ packages/grpc-js/src/resolving-load-balancer.ts | 1 + packages/grpc-js/test/test-resolver.ts | 3 +++ 8 files changed, 29 insertions(+), 7 deletions(-) diff --git a/packages/grpc-js-xds/package.json b/packages/grpc-js-xds/package.json index 3008d3ec..6c734927 100644 --- a/packages/grpc-js-xds/package.json +++ b/packages/grpc-js-xds/package.json @@ -45,7 +45,7 @@ "@grpc/proto-loader": "^0.6.0-pre14" }, "peerDependencies": { - "@grpc/grpc-js": "~1.2.0" + "@grpc/grpc-js": "~1.2.2" }, "engines": { "node": ">=10.10.0" diff --git a/packages/grpc-js-xds/src/resolver-xds.ts b/packages/grpc-js-xds/src/resolver-xds.ts index 814294c8..24d25b97 100644 --- a/packages/grpc-js-xds/src/resolver-xds.ts +++ b/packages/grpc-js-xds/src/resolver-xds.ts @@ -30,8 +30,8 @@ function trace(text: string): void { } class XdsResolver implements Resolver { - private resolutionStarted = false; private hasReportedSuccess = false; + private xdsClient: XdsClient | null = null; constructor( private target: GrpcUri, @@ -51,17 +51,16 @@ class XdsResolver implements Resolver { updateResolution(): void { // Wait until updateResolution is called once to start the xDS requests - if (!this.resolutionStarted) { - this.resolutionStarted = true; + if (this.xdsClient === null) { trace('Starting resolution for target ' + uriToString(this.target)); - const xdsClient = new XdsClient( + this.xdsClient = new XdsClient( this.target.path, { onValidUpdate: (update: ServiceConfig) => { trace('Resolved service config for target ' + uriToString(this.target) + ': ' + JSON.stringify(update)); this.hasReportedSuccess = true; this.listener.onSuccessfulResolution([], update, null, { - xdsClient: xdsClient, + xdsClient: this.xdsClient, }); }, onTransientError: (error: StatusObject) => { @@ -82,6 +81,10 @@ class XdsResolver implements Resolver { } } + destroy() { + this.xdsClient?.shutdown(); + } + static getDefaultAuthority(target: GrpcUri) { return target.path; } diff --git a/packages/grpc-js/package.json b/packages/grpc-js/package.json index 22142dcd..8519f5df 100644 --- a/packages/grpc-js/package.json +++ b/packages/grpc-js/package.json @@ -1,6 +1,6 @@ { "name": "@grpc/grpc-js", - "version": "1.2.1", + "version": "1.2.2", "description": "gRPC Library for Node - pure JS implementation", "homepage": "https://grpc.io/", "repository": "https://github.com/grpc/grpc-node/tree/master/packages/grpc-js", diff --git a/packages/grpc-js/src/resolver-dns.ts b/packages/grpc-js/src/resolver-dns.ts index 2db8a5e4..f19318d3 100644 --- a/packages/grpc-js/src/resolver-dns.ts +++ b/packages/grpc-js/src/resolver-dns.ts @@ -262,6 +262,12 @@ class DnsResolver implements Resolver { } } + destroy() { + /* Do nothing. There is not a practical way to cancel in-flight DNS + * requests, and after this function is called we can expect that + * updateResolution will not be called again. */ + } + /** * Get the default authority for the given target. For IP targets, that is * the IP address. For DNS targets, it is the hostname. diff --git a/packages/grpc-js/src/resolver-uds.ts b/packages/grpc-js/src/resolver-uds.ts index 14bc0176..ed25177c 100644 --- a/packages/grpc-js/src/resolver-uds.ts +++ b/packages/grpc-js/src/resolver-uds.ts @@ -44,6 +44,10 @@ class UdsResolver implements Resolver { ); } + destroy() { + // This resolver owns no resources, so we do nothing here. + } + static getDefaultAuthority(target: GrpcUri): string { return 'localhost'; } diff --git a/packages/grpc-js/src/resolver.ts b/packages/grpc-js/src/resolver.ts index 57c750ae..74f3e514 100644 --- a/packages/grpc-js/src/resolver.ts +++ b/packages/grpc-js/src/resolver.ts @@ -62,6 +62,11 @@ export interface Resolver { * called synchronously with the constructor or updateResolution. */ updateResolution(): void; + + /** + * Destroy the resolver. Should be called when the owning channel shuts down. + */ + destroy(): void; } export interface ResolverConstructor { diff --git a/packages/grpc-js/src/resolving-load-balancer.ts b/packages/grpc-js/src/resolving-load-balancer.ts index 2ce59d0c..5a4c62f5 100644 --- a/packages/grpc-js/src/resolving-load-balancer.ts +++ b/packages/grpc-js/src/resolving-load-balancer.ts @@ -257,6 +257,7 @@ export class ResolvingLoadBalancer implements LoadBalancer { destroy() { this.childLoadBalancer.destroy(); + this.innerResolver.destroy(); this.updateState(ConnectivityState.SHUTDOWN, new UnavailablePicker()); } diff --git a/packages/grpc-js/test/test-resolver.ts b/packages/grpc-js/test/test-resolver.ts index 42ca64d4..c4f42f6e 100644 --- a/packages/grpc-js/test/test-resolver.ts +++ b/packages/grpc-js/test/test-resolver.ts @@ -394,6 +394,9 @@ describe('Name Resolver', () => { return []; } + destroy() { + } + static getDefaultAuthority(target: GrpcUri): string { return 'other'; } From 21da990cb0fdb8130413ea27699f06b321daba87 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Wed, 2 Dec 2020 12:00:19 -0800 Subject: [PATCH 05/46] grpc-js: End calls faster if the deadline has already passed --- packages/grpc-js/package.json | 2 +- packages/grpc-js/src/deadline-filter.ts | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/grpc-js/package.json b/packages/grpc-js/package.json index 8519f5df..17fb1a2e 100644 --- a/packages/grpc-js/package.json +++ b/packages/grpc-js/package.json @@ -1,6 +1,6 @@ { "name": "@grpc/grpc-js", - "version": "1.2.2", + "version": "1.2.3", "description": "gRPC Library for Node - pure JS implementation", "homepage": "https://grpc.io/", "repository": "https://github.com/grpc/grpc-node/tree/master/packages/grpc-js", diff --git a/packages/grpc-js/src/deadline-filter.ts b/packages/grpc-js/src/deadline-filter.ts index 2306bb8f..99bfa2be 100644 --- a/packages/grpc-js/src/deadline-filter.ts +++ b/packages/grpc-js/src/deadline-filter.ts @@ -56,10 +56,14 @@ export class DeadlineFilter extends BaseFilter implements Filter { } const now: number = new Date().getTime(); let timeout = this.deadline - now; - if (timeout < 0) { - timeout = 0; - } - if (this.deadline !== Infinity) { + if (timeout <= 0) { + process.nextTick(() => { + callStream.cancelWithStatus( + Status.DEADLINE_EXCEEDED, + 'Deadline exceeded' + ); + }); + } else if (this.deadline !== Infinity) { this.timer = setTimeout(() => { callStream.cancelWithStatus( Status.DEADLINE_EXCEEDED, From b2530119b90a10dcb15790022477221f47a8f9bc Mon Sep 17 00:00:00 2001 From: d3v53c Date: Mon, 7 Dec 2020 22:29:18 -0800 Subject: [PATCH 06/46] prototype pollution fix --- packages/grpc-js/src/make-client.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/grpc-js/src/make-client.ts b/packages/grpc-js/src/make-client.ts index 05f06b89..a6cb9100 100644 --- a/packages/grpc-js/src/make-client.ts +++ b/packages/grpc-js/src/make-client.ts @@ -93,6 +93,15 @@ export interface ServiceClientConstructor { service: ServiceDefinition; } +/** + * Returns true, if given key is included in the blacklisted + * keys. + * @param key key for check, string. + */ +function isPrototypePolluted(key: string): Boolean { + return ['__proto__', 'prototype', 'constructor'].includes(key); +} + /** * Creates a constructor for a client with the given methods, as specified in * the methods argument. The resulting class will have an instance method for @@ -122,7 +131,7 @@ export function makeClientConstructor( } Object.keys(methods).forEach((name) => { - if (name === '__proto__') { + if (isPrototypePolluted(name)) { return; } const attrs = methods[name]; @@ -155,7 +164,7 @@ export function makeClientConstructor( ServiceClientImpl.prototype[name] = methodFunc; // Associate all provided attributes with the method Object.assign(ServiceClientImpl.prototype[name], attrs); - if (attrs.originalName && attrs.originalName !== '__proto__') { + if (attrs.originalName && !isPrototypePolluted(attrs.originalName)) { ServiceClientImpl.prototype[attrs.originalName] = ServiceClientImpl.prototype[name]; } @@ -204,7 +213,7 @@ export function loadPackageDefinition( if (Object.prototype.hasOwnProperty.call(packageDef, serviceFqn)) { const service = packageDef[serviceFqn]; const nameComponents = serviceFqn.split('.'); - if (nameComponents.some(comp => comp === '__proto__')) { + if (nameComponents.some((comp: string) => isPrototypePolluted(comp))) { continue; } const serviceName = nameComponents[nameComponents.length - 1]; From 61016943973875191ef746b06b9806042d31b96e Mon Sep 17 00:00:00 2001 From: d3v53c Date: Mon, 7 Dec 2020 22:40:14 -0800 Subject: [PATCH 07/46] added test case --- packages/grpc-js/test/test-prototype-pollution.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/grpc-js/test/test-prototype-pollution.ts b/packages/grpc-js/test/test-prototype-pollution.ts index 12092608..6dc4b293 100644 --- a/packages/grpc-js/test/test-prototype-pollution.ts +++ b/packages/grpc-js/test/test-prototype-pollution.ts @@ -24,4 +24,8 @@ describe('loadPackageDefinition', () => { loadPackageDefinition({'__proto__.polluted': true} as any); assert.notStrictEqual(({} as any).polluted, true); }); + it('Should not allow prototype pollution #2', () => { + loadPackageDefinition({'constructor.prototype.polluted': true} as any); + assert.notStrictEqual(({} as any).polluted, true); + }); }); From 374309be66b908a58de667587d6eaa87cef3de93 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Tue, 5 Jan 2021 09:15:17 -0800 Subject: [PATCH 08/46] grpc-js: Propagate internal stream errors from the http2 module --- packages/grpc-js/src/call-stream.ts | 138 +++++++++++++++++----------- 1 file changed, 82 insertions(+), 56 deletions(-) diff --git a/packages/grpc-js/src/call-stream.ts b/packages/grpc-js/src/call-stream.ts index 91e24ea5..9c27aeb1 100644 --- a/packages/grpc-js/src/call-stream.ts +++ b/packages/grpc-js/src/call-stream.ts @@ -37,6 +37,10 @@ const { NGHTTP2_CANCEL, } = http2.constants; +interface NodeError extends Error { + code: string; +} + export type Deadline = Date | number; export interface CallStreamOptions { @@ -202,6 +206,8 @@ export class Http2CallStream implements Call { private listener: InterceptingListener | null = null; + private internalErrorMessage: string | null = null; + constructor( private readonly methodName: string, private readonly channel: ChannelImplementation, @@ -518,66 +524,86 @@ export class Http2CallStream implements Call { this.maybeOutputStatus(); }); stream.on('close', () => { - this.trace('HTTP/2 stream closed with code ' + stream.rstCode); - /* If we have a final status with an OK status code, that means that - * we have received all of the messages and we have processed the - * trailers and the call completed successfully, so it doesn't matter - * how the stream ends after that */ - if (this.finalStatus?.code === Status.OK) { - return; - } - let code: Status; - let details = ''; - switch (stream.rstCode) { - case http2.constants.NGHTTP2_NO_ERROR: - /* If we get a NO_ERROR code and we already have a status, the - * stream completed properly and we just haven't fully processed - * it yet */ - if (this.finalStatus !== null) { - return; - } - code = Status.INTERNAL; - details = `Received RST_STREAM with code ${stream.rstCode}`; - break; - case http2.constants.NGHTTP2_REFUSED_STREAM: - code = Status.UNAVAILABLE; - details = 'Stream refused by server'; - break; - case http2.constants.NGHTTP2_CANCEL: - code = Status.CANCELLED; - details = 'Call cancelled'; - break; - case http2.constants.NGHTTP2_ENHANCE_YOUR_CALM: - code = Status.RESOURCE_EXHAUSTED; - details = 'Bandwidth exhausted'; - break; - case http2.constants.NGHTTP2_INADEQUATE_SECURITY: - code = Status.PERMISSION_DENIED; - details = 'Protocol not secure enough'; - break; - case http2.constants.NGHTTP2_INTERNAL_ERROR: - code = Status.INTERNAL; - /* This error code was previously handled in the default case, and - * there are several instances of it online, so I wanted to - * preserve the original error message so that people find existing - * information in searches, but also include the more recognizable - * "Internal server error" message. */ - details = `Received RST_STREAM with code ${stream.rstCode} (Internal server error)`; - break; - default: - code = Status.INTERNAL; - details = `Received RST_STREAM with code ${stream.rstCode}`; - } - // This is a no-op if trailers were received at all. - // This is OK, because status codes emitted here correspond to more - // catastrophic issues that prevent us from receiving trailers in the - // first place. - this.endCall({ code, details, metadata: new Metadata() }); + /* Use process.next tick to ensure that this code happens after any + * "error" event that may be emitted at about the same time, so that + * we can bubble up the error message from that event. */ + process.nextTick(() => { + this.trace('HTTP/2 stream closed with code ' + stream.rstCode); + /* If we have a final status with an OK status code, that means that + * we have received all of the messages and we have processed the + * trailers and the call completed successfully, so it doesn't matter + * how the stream ends after that */ + if (this.finalStatus?.code === Status.OK) { + return; + } + let code: Status; + let details = ''; + switch (stream.rstCode) { + case http2.constants.NGHTTP2_NO_ERROR: + /* If we get a NO_ERROR code and we already have a status, the + * stream completed properly and we just haven't fully processed + * it yet */ + if (this.finalStatus !== null) { + return; + } + code = Status.INTERNAL; + details = `Received RST_STREAM with code ${stream.rstCode}`; + break; + case http2.constants.NGHTTP2_REFUSED_STREAM: + code = Status.UNAVAILABLE; + details = 'Stream refused by server'; + break; + case http2.constants.NGHTTP2_CANCEL: + code = Status.CANCELLED; + details = 'Call cancelled'; + break; + case http2.constants.NGHTTP2_ENHANCE_YOUR_CALM: + code = Status.RESOURCE_EXHAUSTED; + details = 'Bandwidth exhausted'; + break; + case http2.constants.NGHTTP2_INADEQUATE_SECURITY: + code = Status.PERMISSION_DENIED; + details = 'Protocol not secure enough'; + break; + case http2.constants.NGHTTP2_INTERNAL_ERROR: + code = Status.INTERNAL; + if (this.internalErrorMessage === null) { + /* This error code was previously handled in the default case, and + * there are several instances of it online, so I wanted to + * preserve the original error message so that people find existing + * information in searches, but also include the more recognizable + * "Internal server error" message. */ + details = `Received RST_STREAM with code ${stream.rstCode} (Internal server error)`; + } else { + /* The "Received RST_STREAM with code ..." error is preserved + * here for continuity with errors reported online, but the + * error message at the end will probably be more relevant in + * most cases. */ + details = `Received RST_STREAM with code ${stream.rstCode} triggered by internal client error: ${this.internalErrorMessage}`; + } + break; + default: + code = Status.INTERNAL; + details = `Received RST_STREAM with code ${stream.rstCode}`; + } + // This is a no-op if trailers were received at all. + // This is OK, because status codes emitted here correspond to more + // catastrophic issues that prevent us from receiving trailers in the + // first place. + this.endCall({ code, details, metadata: new Metadata() }); + }); }); - stream.on('error', (err: Error) => { + stream.on('error', (err: NodeError) => { /* We need an error handler here to stop "Uncaught Error" exceptions * from bubbling up. However, errors here should all correspond to * "close" events, where we will handle the error more granularly */ + /* Specifically looking for stream errors that were *not* constructed + * from a RST_STREAM response here: + * https://github.com/nodejs/node/blob/8b8620d580314050175983402dfddf2674e8e22a/lib/internal/http2/core.js#L2267 + */ + if (err.code !== 'ERR_HTTP2_STREAM_ERROR') { + this.internalErrorMessage = err.message; + } }); if (!this.pendingRead) { stream.pause(); From 36986f618a8455ade8ddeeafe5292bbba7104e79 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Tue, 19 Jan 2021 13:43:56 -0800 Subject: [PATCH 09/46] grpc-js: round robin: re-resolve when subchannels go idle --- packages/grpc-js/package.json | 2 +- packages/grpc-js/src/load-balancer-round-robin.ts | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/grpc-js/package.json b/packages/grpc-js/package.json index 17fb1a2e..60d3c908 100644 --- a/packages/grpc-js/package.json +++ b/packages/grpc-js/package.json @@ -1,6 +1,6 @@ { "name": "@grpc/grpc-js", - "version": "1.2.3", + "version": "1.2.4", "description": "gRPC Library for Node - pure JS implementation", "homepage": "https://grpc.io/", "repository": "https://github.com/grpc/grpc-node/tree/master/packages/grpc-js", diff --git a/packages/grpc-js/src/load-balancer-round-robin.ts b/packages/grpc-js/src/load-balancer-round-robin.ts index fc9bef0c..daba4594 100644 --- a/packages/grpc-js/src/load-balancer-round-robin.ts +++ b/packages/grpc-js/src/load-balancer-round-robin.ts @@ -128,14 +128,12 @@ export class RoundRobinLoadBalancer implements LoadBalancer { this.subchannelStateCounts[previousState] -= 1; this.subchannelStateCounts[newState] += 1; this.calculateAndUpdateState(); - - if (newState === ConnectivityState.TRANSIENT_FAILURE) { - this.channelControlHelper.requestReresolution(); - } + if ( newState === ConnectivityState.TRANSIENT_FAILURE || newState === ConnectivityState.IDLE ) { + this.channelControlHelper.requestReresolution(); subchannel.startConnecting(); } }; From b2776b52b4835bafdd0af141b97bfe5cc14b94b9 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Tue, 19 Jan 2021 14:36:41 -0800 Subject: [PATCH 10/46] proto-loader: bump to 0.5.6 --- packages/proto-loader/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/proto-loader/package.json b/packages/proto-loader/package.json index 155a0abf..f0611b2b 100644 --- a/packages/proto-loader/package.json +++ b/packages/proto-loader/package.json @@ -1,6 +1,6 @@ { "name": "@grpc/proto-loader", - "version": "0.5.5", + "version": "0.5.6", "author": "Google Inc.", "contributors": [ { From 85111c2b0bb1e9652ca0a66db23e9a8fc3a6ce70 Mon Sep 17 00:00:00 2001 From: Srini Polavarapu Date: Fri, 22 Jan 2021 10:57:55 -0800 Subject: [PATCH 11/46] Add security policy doc --- SECURITY.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..be6e1087 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +For information on gRPC Security Policy and reporting potentional security issues, please see [gRPC CVE Process](https://github.com/grpc/proposal/blob/master/P4-grpc-cve-process.md). From 5ac9a1c2b617e4ea0dab56a73c60564e9fc5b1bb Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Mon, 25 Jan 2021 13:24:39 -0800 Subject: [PATCH 12/46] grpc-js: Move call to user code out of try block --- packages/grpc-js/package.json | 2 +- packages/grpc-js/src/client-interceptors.ts | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/grpc-js/package.json b/packages/grpc-js/package.json index 60d3c908..01a9d418 100644 --- a/packages/grpc-js/package.json +++ b/packages/grpc-js/package.json @@ -1,6 +1,6 @@ { "name": "@grpc/grpc-js", - "version": "1.2.4", + "version": "1.2.5", "description": "gRPC Library for Node - pure JS implementation", "homepage": "https://grpc.io/", "repository": "https://github.com/grpc/grpc-node/tree/master/packages/grpc-js", diff --git a/packages/grpc-js/src/client-interceptors.ts b/packages/grpc-js/src/client-interceptors.ts index 09e7f5aa..a7cc2f87 100644 --- a/packages/grpc-js/src/client-interceptors.ts +++ b/packages/grpc-js/src/client-interceptors.ts @@ -347,10 +347,11 @@ class BaseInterceptingCall implements InterceptingCallInterface { let serialized: Buffer; try { serialized = this.methodDefinition.requestSerialize(message); - this.call.sendMessageWithContext(context, serialized); } catch (e) { this.call.cancelWithStatus(Status.INTERNAL, `Request message serialization failure: ${e.message}`); + return; } + this.call.sendMessageWithContext(context, serialized); } // eslint-disable-next-line @typescript-eslint/no-explicit-any sendMessage(message: any) { @@ -370,7 +371,6 @@ class BaseInterceptingCall implements InterceptingCallInterface { let deserialized: any; try { deserialized = this.methodDefinition.responseDeserialize(message); - interceptingListener?.onReceiveMessage?.(deserialized); } catch (e) { readError = { code: Status.INTERNAL, @@ -378,7 +378,9 @@ class BaseInterceptingCall implements InterceptingCallInterface { metadata: new Metadata(), }; this.call.cancelWithStatus(readError.code, readError.details); + return; } + interceptingListener?.onReceiveMessage?.(deserialized); }, onReceiveStatus: (status) => { if (readError) { From b011bd069dc75699f0f4e9b6d668278e840331f8 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Wed, 27 Jan 2021 11:15:04 -0800 Subject: [PATCH 13/46] grpc-js-xds: List the files to publish in package.json --- packages/grpc-js-xds/package.json | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/grpc-js-xds/package.json b/packages/grpc-js-xds/package.json index 6c734927..d8c2d48c 100644 --- a/packages/grpc-js-xds/package.json +++ b/packages/grpc-js-xds/package.json @@ -49,5 +49,19 @@ }, "engines": { "node": ">=10.10.0" - } + }, + "files": [ + "src/**/*.ts", + "build/src/**/*.{js,d.ts,js.map}", + "deps/envoy-api/envoy/api/v2/**/*.proto", + "deps/envoy-api/envoy/config/**/*.proto", + "deps/envoy-api/envoy/service/**/*.proto", + "deps/envoy-api/envoy/type/**/*.proto", + "deps/envoy-api/envoy/annotations/**/*.proto", + "deps/googleapis/google/api/**/*.proto", + "deps/googleapis/google/protobuf/**/*.proto", + "deps/googleapis/google/rpc/**/*.proto", + "deps/udpa/udpa/annotations/**/*.proto", + "deps/protoc-gen-validate/validate/**/*.proto" + ] } From 0a98f6295dc1e4fb64f3d1d0f3e005da947ea627 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Wed, 27 Jan 2021 12:16:06 -0800 Subject: [PATCH 14/46] grpc-js-xds: Bubble up xds client initialization errors --- packages/grpc-js-xds/src/resolver-xds.ts | 8 ++++---- packages/grpc-js-xds/src/xds-client.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/grpc-js-xds/src/resolver-xds.ts b/packages/grpc-js-xds/src/resolver-xds.ts index 24d25b97..f9e5264a 100644 --- a/packages/grpc-js-xds/src/resolver-xds.ts +++ b/packages/grpc-js-xds/src/resolver-xds.ts @@ -39,12 +39,12 @@ class XdsResolver implements Resolver { private channelOptions: ChannelOptions ) {} - private reportResolutionError() { + private reportResolutionError(reason: string) { this.listener.onError({ code: status.UNAVAILABLE, details: `xDS name resolution failed for target ${uriToString( this.target - )}`, + )}: ${reason}`, metadata: new Metadata(), }); } @@ -68,12 +68,12 @@ class XdsResolver implements Resolver { * not already provided a ServiceConfig for the upper layer to use */ if (!this.hasReportedSuccess) { trace('Resolution error for target ' + uriToString(this.target) + ' due to xDS client transient error ' + error.details); - this.reportResolutionError(); + this.reportResolutionError(error.details); } }, onResourceDoesNotExist: () => { trace('Resolution error for target ' + uriToString(this.target) + ': resource does not exist'); - this.reportResolutionError(); + this.reportResolutionError("Resource does not exist"); }, }, this.channelOptions diff --git a/packages/grpc-js-xds/src/xds-client.ts b/packages/grpc-js-xds/src/xds-client.ts index da75dc7c..ae67960d 100644 --- a/packages/grpc-js-xds/src/xds-client.ts +++ b/packages/grpc-js-xds/src/xds-client.ts @@ -1155,7 +1155,7 @@ export class XdsClient { } removeClusterWatcher(clusterName: string, watcher: Watcher) { - trace('Watcher removed for endpoint ' + clusterName); + trace('Watcher removed for cluster ' + clusterName); this.adsState[CDS_TYPE_URL].removeWatcher(clusterName, watcher); } From e27e4e02ae97f24e473508867a64cac83a2e2393 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Wed, 27 Jan 2021 14:02:41 -0800 Subject: [PATCH 15/46] Bump grpc-js-xds to 1.2.1 --- packages/grpc-js-xds/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grpc-js-xds/package.json b/packages/grpc-js-xds/package.json index d8c2d48c..f2b32a77 100644 --- a/packages/grpc-js-xds/package.json +++ b/packages/grpc-js-xds/package.json @@ -1,6 +1,6 @@ { "name": "@grpc/grpc-js-xds", - "version": "1.2.0", + "version": "1.2.1", "description": "Plugin for @grpc/grpc-js. Adds the xds:// URL scheme and associated features.", "main": "build/src/index.js", "scripts": { From 8e5f5bc18a0f7d8fac4b85e33ddd49c0f5efa3ff Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Mon, 1 Feb 2021 14:18:24 -0800 Subject: [PATCH 16/46] grpc-js: Add ConfigSelector to Resolver API and plumb it through the channel --- packages/grpc-js-xds/src/resolver-xds.ts | 2 +- packages/grpc-js/src/channel.ts | 66 +++++++++++++++---- packages/grpc-js/src/picker.ts | 1 + packages/grpc-js/src/resolver-dns.ts | 4 +- packages/grpc-js/src/resolver-uds.ts | 1 + packages/grpc-js/src/resolver.ts | 20 +++++- .../grpc-js/src/resolving-load-balancer.ts | 38 ++++++++++- 7 files changed, 113 insertions(+), 19 deletions(-) diff --git a/packages/grpc-js-xds/src/resolver-xds.ts b/packages/grpc-js-xds/src/resolver-xds.ts index 814294c8..e1a0af11 100644 --- a/packages/grpc-js-xds/src/resolver-xds.ts +++ b/packages/grpc-js-xds/src/resolver-xds.ts @@ -60,7 +60,7 @@ class XdsResolver implements Resolver { onValidUpdate: (update: ServiceConfig) => { trace('Resolved service config for target ' + uriToString(this.target) + ': ' + JSON.stringify(update)); this.hasReportedSuccess = true; - this.listener.onSuccessfulResolution([], update, null, { + this.listener.onSuccessfulResolution([], update, null, null, { xdsClient: xdsClient, }); }, diff --git a/packages/grpc-js/src/channel.ts b/packages/grpc-js/src/channel.ts index e1a76c09..7c61aa1f 100644 --- a/packages/grpc-js/src/channel.ts +++ b/packages/grpc-js/src/channel.ts @@ -33,7 +33,7 @@ import { FilterStackFactory } from './filter-stack'; import { CallCredentialsFilterFactory } from './call-credentials-filter'; import { DeadlineFilterFactory } from './deadline-filter'; import { CompressionFilterFactory } from './compression-filter'; -import { getDefaultAuthority, mapUriDefaultScheme } from './resolver'; +import { CallConfig, ConfigSelector, getDefaultAuthority, mapUriDefaultScheme } from './resolver'; import { trace, log } from './logging'; import { SubchannelAddress } from './subchannel'; import { MaxMessageSizeFilterFactory } from './max-message-size-filter'; @@ -136,9 +136,18 @@ export class ChannelImplementation implements Channel { private subchannelPool: SubchannelPool; private connectivityState: ConnectivityState = ConnectivityState.IDLE; private currentPicker: Picker = new UnavailablePicker(); + /** + * Calls queued up to get a call config. Should only be populated before the + * first time the resolver returns a result, which includes the ConfigSelector. + */ + private configSelectionQueue: Array<{ + callStream: Http2CallStream; + callMetadata: Metadata; + }> = []; private pickQueue: Array<{ callStream: Http2CallStream; callMetadata: Metadata; + callConfig: CallConfig; }> = []; private connectivityStateWatchers: ConnectivityStateWatcher[] = []; private defaultAuthority: string; @@ -152,6 +161,7 @@ export class ChannelImplementation implements Channel { * is non-empty. */ private callRefTimer: NodeJS.Timer; + private configSelector: ConfigSelector | null = null; constructor( target: string, private readonly credentials: ChannelCredentials, @@ -227,8 +237,8 @@ export class ChannelImplementation implements Channel { const queueCopy = this.pickQueue.slice(); this.callRefTimer.unref?.(); this.pickQueue = []; - for (const { callStream, callMetadata } of queueCopy) { - this.tryPick(callStream, callMetadata); + for (const { callStream, callMetadata, callConfig } of queueCopy) { + this.tryPick(callStream, callMetadata, callConfig); } this.updateState(connectivityState); }, @@ -242,7 +252,17 @@ export class ChannelImplementation implements Channel { this.resolvingLoadBalancer = new ResolvingLoadBalancer( this.target, channelControlHelper, - options + options, + (configSelector) => { + this.configSelector = configSelector; + /* We process the queue asynchronously to ensure that the corresponding + * load balancer update has completed. */ + process.nextTick(() => { + for (const {callStream, callMetadata} of this.configSelectionQueue) { + this.tryGetConfig(callStream, callMetadata); + } + }); + } ); this.filterStackFactory = new FilterStackFactory([ new CallCredentialsFilterFactory(this), @@ -252,9 +272,9 @@ export class ChannelImplementation implements Channel { ]); } - private pushPick(callStream: Http2CallStream, callMetadata: Metadata) { + private pushPick(callStream: Http2CallStream, callMetadata: Metadata, callConfig: CallConfig) { this.callRefTimer.ref?.(); - this.pickQueue.push({ callStream, callMetadata }); + this.pickQueue.push({ callStream, callMetadata, callConfig }); } /** @@ -264,8 +284,8 @@ export class ChannelImplementation implements Channel { * @param callStream * @param callMetadata */ - private tryPick(callStream: Http2CallStream, callMetadata: Metadata) { - const pickResult = this.currentPicker.pick({ metadata: callMetadata }); + private tryPick(callStream: Http2CallStream, callMetadata: Metadata, callConfig: CallConfig) { + const pickResult = this.currentPicker.pick({ metadata: callMetadata, extraPickInfo: callConfig.pickInformation }); trace( LogVerbosity.DEBUG, 'channel', @@ -301,7 +321,7 @@ export class ChannelImplementation implements Channel { ' has state ' + ConnectivityState[pickResult.subchannel!.getConnectivityState()] ); - this.pushPick(callStream, callMetadata); + this.pushPick(callStream, callMetadata, callConfig); break; } /* We need to clone the callMetadata here because the transparent @@ -321,6 +341,7 @@ export class ChannelImplementation implements Channel { ); /* If we reach this point, the call stream has started * successfully */ + callConfig.onCommitted?.(); pickResult.onCallStarted?.(); } catch (error) { if ( @@ -349,7 +370,7 @@ export class ChannelImplementation implements Channel { (error as Error).message + '. Retrying pick' ); - this.tryPick(callStream, callMetadata); + this.tryPick(callStream, callMetadata, callConfig); } else { trace( LogVerbosity.INFO, @@ -378,7 +399,7 @@ export class ChannelImplementation implements Channel { ConnectivityState[subchannelState] + ' after metadata filters. Retrying pick' ); - this.tryPick(callStream, callMetadata); + this.tryPick(callStream, callMetadata, callConfig); } }, (error: Error & { code: number }) => { @@ -392,11 +413,11 @@ export class ChannelImplementation implements Channel { } break; case PickResultType.QUEUE: - this.pushPick(callStream, callMetadata); + this.pushPick(callStream, callMetadata, callConfig); break; case PickResultType.TRANSIENT_FAILURE: if (callMetadata.getOptions().waitForReady) { - this.pushPick(callStream, callMetadata); + this.pushPick(callStream, callMetadata, callConfig); } else { callStream.cancelWithStatus( pickResult.status!.code, @@ -451,8 +472,25 @@ export class ChannelImplementation implements Channel { } } + private tryGetConfig(stream: Http2CallStream, metadata: Metadata) { + if (this.configSelector === null) { + this.callRefTimer.ref?.(); + this.configSelectionQueue.push({ + callStream: stream, + callMetadata: metadata + }); + } else { + const callConfig = this.configSelector(stream.getMethod(), metadata); + if (callConfig.status === Status.OK) { + this.tryPick(stream, metadata, callConfig); + } else { + stream.cancelWithStatus(callConfig.status, "Failed to route call to method " + stream.getMethod()); + } + } + } + _startCallStream(stream: Http2CallStream, metadata: Metadata) { - this.tryPick(stream, metadata.clone()); + this.tryGetConfig(stream, metadata.clone()); } close() { diff --git a/packages/grpc-js/src/picker.ts b/packages/grpc-js/src/picker.ts index 184047b2..6df61b59 100644 --- a/packages/grpc-js/src/picker.ts +++ b/packages/grpc-js/src/picker.ts @@ -85,6 +85,7 @@ export interface DropCallPickResult extends PickResult { export interface PickArgs { metadata: Metadata; + extraPickInfo: {[key: string]: string}; } /** diff --git a/packages/grpc-js/src/resolver-dns.ts b/packages/grpc-js/src/resolver-dns.ts index 2db8a5e4..96d78f8a 100644 --- a/packages/grpc-js/src/resolver-dns.ts +++ b/packages/grpc-js/src/resolver-dns.ts @@ -129,7 +129,7 @@ class DnsResolver implements Resolver { if (this.ipResult !== null) { trace('Returning IP address for target ' + uriToString(this.target)); setImmediate(() => { - this.listener.onSuccessfulResolution(this.ipResult!, null, null, {}); + this.listener.onSuccessfulResolution(this.ipResult!, null, null, null, {}); }); return; } @@ -192,6 +192,7 @@ class DnsResolver implements Resolver { this.latestLookupResult, this.latestServiceConfig, this.latestServiceConfigError, + null, {} ); }, @@ -237,6 +238,7 @@ class DnsResolver implements Resolver { this.latestLookupResult, this.latestServiceConfig, this.latestServiceConfigError, + null, {} ); } diff --git a/packages/grpc-js/src/resolver-uds.ts b/packages/grpc-js/src/resolver-uds.ts index 14bc0176..e7667c78 100644 --- a/packages/grpc-js/src/resolver-uds.ts +++ b/packages/grpc-js/src/resolver-uds.ts @@ -40,6 +40,7 @@ class UdsResolver implements Resolver { this.addresses, null, null, + null, {} ); } diff --git a/packages/grpc-js/src/resolver.ts b/packages/grpc-js/src/resolver.ts index 57c750ae..e9176234 100644 --- a/packages/grpc-js/src/resolver.ts +++ b/packages/grpc-js/src/resolver.ts @@ -15,13 +15,30 @@ * */ -import { ServiceConfig } from './service-config'; +import { MethodConfig, ServiceConfig } from './service-config'; import * as resolver_dns from './resolver-dns'; import * as resolver_uds from './resolver-uds'; import { StatusObject } from './call-stream'; import { SubchannelAddress } from './subchannel'; import { GrpcUri, uriToString } from './uri-parser'; import { ChannelOptions } from './channel-options'; +import { Metadata } from './metadata'; +import { Status } from './constants'; + +export interface CallConfig { + methodConfig: MethodConfig; + onCommitted?: () => void; + pickInformation: {[key: string]: string}; + status: Status; +} + +/** + * Selects a configuration for a method given the name and metadata. Defined in + * https://github.com/grpc/proposal/blob/master/A31-xds-timeout-support-and-config-selector.md#new-functionality-in-grpc + */ +export interface ConfigSelector { + (methodName: string, metadata: Metadata): CallConfig; +} /** * A listener object passed to the resolver's constructor that provides name @@ -41,6 +58,7 @@ export interface ResolverListener { addressList: SubchannelAddress[], serviceConfig: ServiceConfig | null, serviceConfigError: StatusObject | null, + configSelector: ConfigSelector | null, attributes: { [key: string]: unknown } ): void; /** diff --git a/packages/grpc-js/src/resolving-load-balancer.ts b/packages/grpc-js/src/resolving-load-balancer.ts index 2ce59d0c..039a17b0 100644 --- a/packages/grpc-js/src/resolving-load-balancer.ts +++ b/packages/grpc-js/src/resolving-load-balancer.ts @@ -23,7 +23,7 @@ import { } from './load-balancer'; import { ServiceConfig, validateServiceConfig } from './service-config'; import { ConnectivityState } from './channel'; -import { createResolver, Resolver } from './resolver'; +import { ConfigSelector, createResolver, Resolver } from './resolver'; import { ServiceError } from './call'; import { Picker, UnavailablePicker, QueuePicker } from './picker'; import { BackoffTimeout } from './backoff-timeout'; @@ -46,6 +46,36 @@ function trace(text: string): void { const DEFAULT_LOAD_BALANCER_NAME = 'pick_first'; +function getDefaultConfigSelector(serviceConfig: ServiceConfig | null): ConfigSelector { + return function defaultConfigSelector(methodName: string, metadata: Metadata) { + const splitName = methodName.split('/').filter(x => x.length > 0); + const service = splitName[0] ?? ''; + const method = splitName[1] ?? ''; + if (serviceConfig && serviceConfig.methodConfig) { + for (const methodConfig of serviceConfig.methodConfig) { + for (const name of methodConfig.name) { + if (name.service === service && (name.method === undefined || name.method === method)) { + return { + methodConfig: methodConfig, + pickInformation: {}, + status: Status.OK + }; + } + } + } + } + return { + methodConfig: {name: []}, + pickInformation: {}, + status: Status.OK + }; + } +} + +export interface ResolutionCallback { + (configSelector: ConfigSelector): void; +} + export class ResolvingLoadBalancer implements LoadBalancer { /** * The resolver class constructed for the target address. @@ -93,7 +123,8 @@ export class ResolvingLoadBalancer implements LoadBalancer { constructor( private readonly target: GrpcUri, private readonly channelControlHelper: ChannelControlHelper, - private readonly channelOptions: ChannelOptions + private readonly channelOptions: ChannelOptions, + private readonly onSuccessfulResolution: ResolutionCallback ) { if (channelOptions['grpc.service_config']) { this.defaultServiceConfig = validateServiceConfig( @@ -134,6 +165,7 @@ export class ResolvingLoadBalancer implements LoadBalancer { addressList: SubchannelAddress[], serviceConfig: ServiceConfig | null, serviceConfigError: ServiceError | null, + configSelector: ConfigSelector | null, attributes: { [key: string]: unknown } ) => { let workingServiceConfig: ServiceConfig | null = null; @@ -180,6 +212,8 @@ export class ResolvingLoadBalancer implements LoadBalancer { loadBalancingConfig, attributes ); + const finalServiceConfig = workingServiceConfig ?? this.defaultServiceConfig; + this.onSuccessfulResolution(configSelector ?? getDefaultConfigSelector(finalServiceConfig)); }, onError: (error: StatusObject) => { this.handleResolutionFailure(error); From 887d2ef677e66a0a1a6e6cca1faa6467ad252e96 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Tue, 2 Feb 2021 14:16:10 -0800 Subject: [PATCH 17/46] Kick the ResolvingLoadBalancer out of IDLE when the first call is started. --- packages/grpc-js/src/channel.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/grpc-js/src/channel.ts b/packages/grpc-js/src/channel.ts index 7c61aa1f..fd015aff 100644 --- a/packages/grpc-js/src/channel.ts +++ b/packages/grpc-js/src/channel.ts @@ -261,6 +261,7 @@ export class ChannelImplementation implements Channel { for (const {callStream, callMetadata} of this.configSelectionQueue) { this.tryGetConfig(callStream, callMetadata); } + this.configSelectionQueue = []; }); } ); @@ -474,6 +475,11 @@ export class ChannelImplementation implements Channel { private tryGetConfig(stream: Http2CallStream, metadata: Metadata) { if (this.configSelector === null) { + /* This branch will only be taken at the beginning of the channel's life, + * before the resolver ever returns a result. So, the + * ResolvingLoadBalancer may be idle and if so it needs to be kicked + * because it now has a pending request. */ + this.resolvingLoadBalancer.exitIdle(); this.callRefTimer.ref?.(); this.configSelectionQueue.push({ callStream: stream, From 3806a99760c901f6a0cb40a1f3b899434555a45d Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Mon, 8 Feb 2021 09:08:41 -0800 Subject: [PATCH 18/46] Add handling for early name resolution failures --- packages/grpc-js/src/channel.ts | 41 +++++++++++++++++-- .../grpc-js/src/resolving-load-balancer.ts | 8 +++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/packages/grpc-js/src/channel.ts b/packages/grpc-js/src/channel.ts index fd015aff..6f614233 100644 --- a/packages/grpc-js/src/channel.ts +++ b/packages/grpc-js/src/channel.ts @@ -235,8 +235,8 @@ export class ChannelImplementation implements Channel { updateState: (connectivityState: ConnectivityState, picker: Picker) => { this.currentPicker = picker; const queueCopy = this.pickQueue.slice(); - this.callRefTimer.unref?.(); this.pickQueue = []; + this.callRefTimerUnref(); for (const { callStream, callMetadata, callConfig } of queueCopy) { this.tryPick(callStream, callMetadata, callConfig); } @@ -258,11 +258,30 @@ export class ChannelImplementation implements Channel { /* We process the queue asynchronously to ensure that the corresponding * load balancer update has completed. */ process.nextTick(() => { - for (const {callStream, callMetadata} of this.configSelectionQueue) { + const localQueue = this.configSelectionQueue; + this.configSelectionQueue = []; + this.callRefTimerUnref() + for (const {callStream, callMetadata} of localQueue) { this.tryGetConfig(callStream, callMetadata); } this.configSelectionQueue = []; }); + }, + (status) => { + if (this.configSelectionQueue.length > 0) { + trace(LogVerbosity.DEBUG, 'channel', 'Name resolution failed for target ' + uriToString(this.target) + ' with calls queued for config selection'); + } + const localQueue = this.configSelectionQueue; + this.configSelectionQueue = []; + this.callRefTimerUnref(); + for (const {callStream, callMetadata} of localQueue) { + if (callMetadata.getOptions().waitForReady) { + this.callRefTimerRef(); + this.configSelectionQueue.push({callStream, callMetadata}); + } else { + callStream.cancelWithStatus(status.code, status.details); + } + } } ); this.filterStackFactory = new FilterStackFactory([ @@ -273,9 +292,23 @@ export class ChannelImplementation implements Channel { ]); } + private callRefTimerRef() { + if (!this.callRefTimer.hasRef()) { + trace(LogVerbosity.DEBUG, 'channel', 'callRefTimer.ref | configSelectionQueue.length=' + this.configSelectionQueue.length + ' pickQueue.length=' + this.pickQueue.length); + this.callRefTimer.ref?.(); + } + } + + private callRefTimerUnref() { + if (this.callRefTimer.hasRef()) { + trace(LogVerbosity.DEBUG, 'channel', 'callRefTimer.unref | configSelectionQueue.length=' + this.configSelectionQueue.length + ' pickQueue.length=' + this.pickQueue.length); + this.callRefTimer.unref?.(); + } + } + private pushPick(callStream: Http2CallStream, callMetadata: Metadata, callConfig: CallConfig) { - this.callRefTimer.ref?.(); this.pickQueue.push({ callStream, callMetadata, callConfig }); + this.callRefTimerRef(); } /** @@ -480,11 +513,11 @@ export class ChannelImplementation implements Channel { * ResolvingLoadBalancer may be idle and if so it needs to be kicked * because it now has a pending request. */ this.resolvingLoadBalancer.exitIdle(); - this.callRefTimer.ref?.(); this.configSelectionQueue.push({ callStream: stream, callMetadata: metadata }); + this.callRefTimerRef(); } else { const callConfig = this.configSelector(stream.getMethod(), metadata); if (callConfig.status === Status.OK) { diff --git a/packages/grpc-js/src/resolving-load-balancer.ts b/packages/grpc-js/src/resolving-load-balancer.ts index 7cf7e36d..84fe4ae1 100644 --- a/packages/grpc-js/src/resolving-load-balancer.ts +++ b/packages/grpc-js/src/resolving-load-balancer.ts @@ -76,6 +76,10 @@ export interface ResolutionCallback { (configSelector: ConfigSelector): void; } +export interface ResolutionFailureCallback { + (status: StatusObject): void; +} + export class ResolvingLoadBalancer implements LoadBalancer { /** * The resolver class constructed for the target address. @@ -124,7 +128,8 @@ export class ResolvingLoadBalancer implements LoadBalancer { private readonly target: GrpcUri, private readonly channelControlHelper: ChannelControlHelper, private readonly channelOptions: ChannelOptions, - private readonly onSuccessfulResolution: ResolutionCallback + private readonly onSuccessfulResolution: ResolutionCallback, + private readonly onFailedResolution: ResolutionFailureCallback ) { if (channelOptions['grpc.service_config']) { this.defaultServiceConfig = validateServiceConfig( @@ -261,6 +266,7 @@ export class ResolvingLoadBalancer implements LoadBalancer { ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker(error) ); + this.onFailedResolution(error); } this.backoffTimeout.runOnce(); } From 9e084bce1965d5dfa0a14564a0cf66d05c005724 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Mon, 8 Feb 2021 11:39:15 -0800 Subject: [PATCH 19/46] Handle absence of Timer#hasRef on older Node versions --- packages/grpc-js/src/channel.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/grpc-js/src/channel.ts b/packages/grpc-js/src/channel.ts index 6f614233..dad8a532 100644 --- a/packages/grpc-js/src/channel.ts +++ b/packages/grpc-js/src/channel.ts @@ -293,14 +293,16 @@ export class ChannelImplementation implements Channel { } private callRefTimerRef() { - if (!this.callRefTimer.hasRef()) { + // If the hasRef function does not exist, always run the code + if (!this.callRefTimer.hasRef?.()) { trace(LogVerbosity.DEBUG, 'channel', 'callRefTimer.ref | configSelectionQueue.length=' + this.configSelectionQueue.length + ' pickQueue.length=' + this.pickQueue.length); this.callRefTimer.ref?.(); } } private callRefTimerUnref() { - if (this.callRefTimer.hasRef()) { + // If the hasRef function does not exist, always run the code + if ((!this.callRefTimer.hasRef) || (this.callRefTimer.hasRef())) { trace(LogVerbosity.DEBUG, 'channel', 'callRefTimer.unref | configSelectionQueue.length=' + this.configSelectionQueue.length + ' pickQueue.length=' + this.pickQueue.length); this.callRefTimer.unref?.(); } From c3c39af8ac324c73fd46b44596b1fc3e41199141 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Tue, 9 Feb 2021 12:10:04 -0800 Subject: [PATCH 20/46] grpc-js-xds: Add XdsClusterManager LB policy --- packages/grpc-js-xds/src/index.ts | 2 + .../src/load-balancer-xds-cluster-manager.ts | 286 ++++++++++++++++++ 2 files changed, 288 insertions(+) create mode 100644 packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts diff --git a/packages/grpc-js-xds/src/index.ts b/packages/grpc-js-xds/src/index.ts index 06bea990..1b24d25e 100644 --- a/packages/grpc-js-xds/src/index.ts +++ b/packages/grpc-js-xds/src/index.ts @@ -21,6 +21,7 @@ import * as load_balancer_eds from './load-balancer-eds'; import * as load_balancer_lrs from './load-balancer-lrs'; import * as load_balancer_priority from './load-balancer-priority'; import * as load_balancer_weighted_target from './load-balancer-weighted-target'; +import * as load_balancer_xds_cluster_manager from './load-balancer-xds-cluster-manager'; /** * Register the "xds:" name scheme with the @grpc/grpc-js library. @@ -32,4 +33,5 @@ export function register() { load_balancer_lrs.setup(); load_balancer_priority.setup(); load_balancer_weighted_target.setup(); + load_balancer_xds_cluster_manager.setup(); } \ No newline at end of file diff --git a/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts b/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts new file mode 100644 index 00000000..c4f53759 --- /dev/null +++ b/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts @@ -0,0 +1,286 @@ +/* + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { connectivityState as ConnectivityState, status as Status, experimental, logVerbosity, Metadata, status } from "@grpc/grpc-js/"; + +import LoadBalancingConfig = experimental.LoadBalancingConfig; +import validateLoadBalancingConfig = experimental.validateLoadBalancingConfig; +import LoadBalancer = experimental.LoadBalancer; +import Picker = experimental.Picker; +import PickResult = experimental.PickResult; +import PickArgs = experimental.PickArgs; +import PickResultType = experimental.PickResultType; +import UnavailablePicker = experimental.UnavailablePicker; +import QueuePicker = experimental.QueuePicker; +import SubchannelAddress = experimental.SubchannelAddress; +import ChildLoadBalancerHandler = experimental.ChildLoadBalancerHandler; +import getFirstUsableConfig = experimental.getFirstUsableConfig; +import ChannelControlHelper = experimental.ChannelControlHelper; +import registerLoadBalancerType = experimental.registerLoadBalancerType; + +const TRACER_NAME = 'weighted_target'; + +function trace(text: string): void { + experimental.trace(logVerbosity.DEBUG, TRACER_NAME, text); +} + +const TYPE_NAME = 'xds_cluster_manager'; + +interface ClusterManagerChild { + child_policy: LoadBalancingConfig[]; +} + +class XdsClusterManagerLoadBalancingConfig implements LoadBalancingConfig { + getLoadBalancerName(): string { + return TYPE_NAME; + } + + constructor(private children: Map) {} + + getChildren() { + return this.children; + } + + toJsonObject(): object { + const childrenField: {[key: string]: object} = {}; + for (const [childName, childValue] of this.children.entries()) { + childrenField[childName] = { + child_policy: childValue.child_policy.map(policy => policy.toJsonObject()) + }; + } + return { + [TYPE_NAME]: { + children: childrenField + } + } + } + + static createFromJson(obj: any): XdsClusterManagerLoadBalancingConfig { + const childrenMap: Map = new Map(); + if (!('children' in obj && obj.children !== null && typeof obj.children === 'object')) { + throw new Error('xds_cluster_manager config must have a children map'); + } + for (const key of obj.children) { + const childObj = obj.children[key]; + if (!('child_policy' in childObj && Array.isArray(childObj.child_policy))) { + throw new Error(`xds_cluster_manager child ${key} must have a child_policy array`); + } + const validatedChild = { + child_policy: childObj.child_policy.map(validateLoadBalancingConfig) + }; + childrenMap.set(key, validatedChild); + } + return new XdsClusterManagerLoadBalancingConfig(childrenMap); + } +} + +class XdsClusterManagerPicker implements Picker { + constructor(private childPickers: Map) {} + + pick(pickArgs: PickArgs): PickResult { + /* extraPickInfo.cluster should be set for all calls by the config selector + * corresponding to the service config that specified the use of this LB + * policy. */ + const cluster = pickArgs.extraPickInfo.cluster ?? ''; + if (this.childPickers.has(cluster)) { + return this.childPickers.get(cluster)!.pick(pickArgs); + } else { + return { + pickResultType: PickResultType.TRANSIENT_FAILURE, + status: { + code: status.INTERNAL, + details: `Requested cluster ${cluster} not found`, + metadata: new Metadata(), + }, + subchannel: null, + extraFilterFactory: null, + onCallStarted: null + }; + } + } +} + +interface XdsClusterManagerChild { + updateAddressList(addressList: SubchannelAddress[], lbConfig: ClusterManagerChild, attributes: { [key: string]: unknown; }): void; + exitIdle(): void; + resetBackoff(): void; + destroy(): void; + getConnectivityState(): ConnectivityState; + getPicker(): Picker; + +} + +class XdsClusterManager implements LoadBalancer { + private XdsClusterManagerChildImpl = class implements XdsClusterManagerChild { + private connectivityState: ConnectivityState = ConnectivityState.IDLE; + private picker: Picker; + private childBalancer: ChildLoadBalancerHandler; + + constructor(private parent: XdsClusterManager, private name: string) { + this.childBalancer = new ChildLoadBalancerHandler({ + createSubchannel: (subchannelAddress, subchannelOptions) => { + return this.parent.channelControlHelper.createSubchannel(subchannelAddress, subchannelOptions); + }, + updateState: (connectivityState, picker) => { + this.updateState(connectivityState, picker); + }, + requestReresolution: () => { + this.parent.channelControlHelper.requestReresolution(); + } + }); + + this.picker = new QueuePicker(this.childBalancer); + } + + private updateState(connectivityState: ConnectivityState, picker: Picker) { + trace('Child ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]); + this.connectivityState = connectivityState; + this.picker = picker; + this.parent.updateState(); + } + updateAddressList(addressList: SubchannelAddress[], lbConfig: ClusterManagerChild, attributes: { [key: string]: unknown; }): void { + const childConfig = getFirstUsableConfig(lbConfig.child_policy); + if (childConfig !== null) { + this.childBalancer.updateAddressList(addressList, childConfig, attributes); + } + } + exitIdle(): void { + this.childBalancer.exitIdle(); + } + resetBackoff(): void { + this.childBalancer.resetBackoff(); + } + destroy(): void { + this.childBalancer.destroy(); + } + getConnectivityState(): ConnectivityState { + return this.connectivityState; + } + getPicker(): Picker { + return this.picker; + } + } + // End of XdsClusterManagerChildImpl + + private children: Map = new Map(); + constructor(private channelControlHelper: ChannelControlHelper) {} + + private updateState() { + const pickerMap: Map = new Map(); + let anyReady = false; + let anyConnecting = false; + let anyIdle = false; + for (const [name, child] of this.children.entries()) { + pickerMap.set(name, child.getPicker()); + switch (child.getConnectivityState()) { + case ConnectivityState.READY: + anyReady = true; + break; + case ConnectivityState.CONNECTING: + anyConnecting = true; + break; + case ConnectivityState.IDLE: + anyIdle = true; + break; + } + } + let connectivityState: ConnectivityState; + if (anyReady) { + connectivityState = ConnectivityState.READY; + } else if (anyConnecting) { + connectivityState = ConnectivityState.CONNECTING; + } else if (anyIdle) { + connectivityState = ConnectivityState.IDLE; + } else { + connectivityState = ConnectivityState.TRANSIENT_FAILURE; + } + let picker: Picker; + + switch (connectivityState) { + case ConnectivityState.READY: + picker = new XdsClusterManagerPicker(pickerMap); + break; + case ConnectivityState.CONNECTING: + case ConnectivityState.IDLE: + picker = new QueuePicker(this); + break; + default: + picker = new UnavailablePicker({ + code: Status.UNAVAILABLE, + details: 'xds_cluster_manager: all children report state TRANSIENT_FAILURE', + metadata: new Metadata() + }); + } + trace( + 'Transitioning to ' + + ConnectivityState[connectivityState] + ); + this.channelControlHelper.updateState(connectivityState, picker); + } + + updateAddressList(addressList: SubchannelAddress[], lbConfig: LoadBalancingConfig, attributes: { [key: string]: unknown; }): void { + if (!(lbConfig instanceof XdsClusterManagerLoadBalancingConfig)) { + // Reject a config of the wrong type + trace('Discarding address list update with unrecognized config ' + JSON.stringify(lbConfig.toJsonObject(), undefined, 2)); + return; + } + const configChildren = lbConfig.getChildren(); + // Delete children that are not in the new config + const namesToRemove: string[] = []; + for (const name of this.children.keys()) { + if (!configChildren.has(name)) { + namesToRemove.push(name); + } + } + for (const name of namesToRemove) { + this.children.get(name)!.destroy(); + this.children.delete(name); + } + // Add new children that were not in the previous config + for (const [name, childConfig] of configChildren.entries()) { + if (!this.children.has(name)) { + const newChild = new this.XdsClusterManagerChildImpl(this, name); + newChild.updateAddressList(addressList, childConfig, attributes); + this.children.set(name, newChild); + } + } + this.updateState(); + } + exitIdle(): void { + for (const child of this.children.values()) { + child.exitIdle(); + } + } + resetBackoff(): void { + for (const child of this.children.values()) { + child.resetBackoff(); + } + } + destroy(): void { + for (const child of this.children.values()) { + child.destroy(); + } + this.children.clear(); + } + getTypeName(): string { + return TYPE_NAME; + } +} + +export function setup() { + registerLoadBalancerType(TYPE_NAME, XdsClusterManager, XdsClusterManagerLoadBalancingConfig); +} \ No newline at end of file From d1aa9aa6fc02ad7152a7495f43d3769b0a18dfc7 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Tue, 9 Feb 2021 12:20:01 -0800 Subject: [PATCH 21/46] Don't update identical states with identical pickers --- .../grpc-js-xds/src/load-balancer-xds-cluster-manager.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts b/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts index c4f53759..f8d2de25 100644 --- a/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts +++ b/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts @@ -177,6 +177,8 @@ class XdsClusterManager implements LoadBalancer { // End of XdsClusterManagerChildImpl private children: Map = new Map(); + // Shutdown is a placeholder value that will never appear in normal operation. + private currentState: ConnectivityState = ConnectivityState.SHUTDOWN; constructor(private channelControlHelper: ChannelControlHelper) {} private updateState() { @@ -208,6 +210,13 @@ class XdsClusterManager implements LoadBalancer { } else { connectivityState = ConnectivityState.TRANSIENT_FAILURE; } + /* For each of the states CONNECTING, IDLE, and TRANSIENT_FAILURE, there is + * exactly one corresponding picker, so if the state is one of those and + * that does not change, no new information is provided by passing the + * new state upward. */ + if (connectivityState === this.currentState && connectivityState !== ConnectivityState.READY) { + return; + } let picker: Picker; switch (connectivityState) { From c953a0e212b16e3d269ce3a661799caaf274039e Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Fri, 12 Feb 2021 13:37:52 -0800 Subject: [PATCH 22/46] refactor part of xds-client into seprate files --- packages/grpc-js-xds/src/load-balancer-cds.ts | 3 +- .../src/xds-stream-state/cds-state.ts | 171 +++++++++++++++++ .../src/xds-stream-state/eds-state.ts | 174 ++++++++++++++++++ .../src/xds-stream-state/lds-state.ts | 105 +++++++++++ .../src/xds-stream-state/rds-state.ts | 94 ++++++++++ .../src/xds-stream-state/xds-stream-state.ts | 38 ++++ 6 files changed, 584 insertions(+), 1 deletion(-) create mode 100644 packages/grpc-js-xds/src/xds-stream-state/cds-state.ts create mode 100644 packages/grpc-js-xds/src/xds-stream-state/eds-state.ts create mode 100644 packages/grpc-js-xds/src/xds-stream-state/lds-state.ts create mode 100644 packages/grpc-js-xds/src/xds-stream-state/rds-state.ts create mode 100644 packages/grpc-js-xds/src/xds-stream-state/xds-stream-state.ts diff --git a/packages/grpc-js-xds/src/load-balancer-cds.ts b/packages/grpc-js-xds/src/load-balancer-cds.ts index a2961927..452b1304 100644 --- a/packages/grpc-js-xds/src/load-balancer-cds.ts +++ b/packages/grpc-js-xds/src/load-balancer-cds.ts @@ -16,7 +16,7 @@ */ import { connectivityState, status, Metadata, logVerbosity, experimental } from '@grpc/grpc-js'; -import { XdsClient, Watcher } from './xds-client'; +import { XdsClient } from './xds-client'; import { Cluster__Output } from './generated/envoy/api/v2/Cluster'; import SubchannelAddress = experimental.SubchannelAddress; import UnavailablePicker = experimental.UnavailablePicker; @@ -26,6 +26,7 @@ import ChannelControlHelper = experimental.ChannelControlHelper; import registerLoadBalancerType = experimental.registerLoadBalancerType; import LoadBalancingConfig = experimental.LoadBalancingConfig; import { EdsLoadBalancingConfig } from './load-balancer-eds'; +import { Watcher } from './xds-stream-state/xds-stream-state'; const TRACER_NAME = 'cds_balancer'; diff --git a/packages/grpc-js-xds/src/xds-stream-state/cds-state.ts b/packages/grpc-js-xds/src/xds-stream-state/cds-state.ts new file mode 100644 index 00000000..34308995 --- /dev/null +++ b/packages/grpc-js-xds/src/xds-stream-state/cds-state.ts @@ -0,0 +1,171 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { experimental, logVerbosity, StatusObject } from "@grpc/grpc-js"; +import { Cluster__Output } from "../generated/envoy/api/v2/Cluster"; +import { EdsState } from "./eds-state"; +import { Watcher, XdsStreamState } from "./xds-stream-state"; + +const TRACER_NAME = 'xds_client'; + +function trace(text: string): void { + experimental.trace(logVerbosity.DEBUG, TRACER_NAME, text); +} + +export class CdsState implements XdsStreamState { + versionInfo = ''; + nonce = ''; + + private watchers: Map[]> = new Map< + string, + Watcher[] + >(); + + private latestResponses: Cluster__Output[] = []; + + constructor( + private edsState: EdsState, + private updateResourceNames: () => void + ) {} + + /** + * Add the watcher to the watcher list. Returns true if the list of resource + * names has changed, and false otherwise. + * @param clusterName + * @param watcher + */ + addWatcher(clusterName: string, watcher: Watcher): void { + trace('Adding CDS watcher for clusterName ' + clusterName); + let watchersEntry = this.watchers.get(clusterName); + let addedServiceName = false; + if (watchersEntry === undefined) { + addedServiceName = true; + watchersEntry = []; + this.watchers.set(clusterName, watchersEntry); + } + watchersEntry.push(watcher); + + /* If we have already received an update for the requested edsServiceName, + * immediately pass that update along to the watcher */ + for (const message of this.latestResponses) { + if (message.name === clusterName) { + /* These updates normally occur asynchronously, so we ensure that + * the same happens here */ + process.nextTick(() => { + trace('Reporting existing CDS update for new watcher for clusterName ' + clusterName); + watcher.onValidUpdate(message); + }); + } + } + if (addedServiceName) { + this.updateResourceNames(); + } + } + + removeWatcher(clusterName: string, watcher: Watcher): void { + trace('Removing CDS watcher for clusterName ' + clusterName); + const watchersEntry = this.watchers.get(clusterName); + let removedServiceName = false; + if (watchersEntry !== undefined) { + const entryIndex = watchersEntry.indexOf(watcher); + if (entryIndex >= 0) { + watchersEntry.splice(entryIndex, 1); + } + if (watchersEntry.length === 0) { + removedServiceName = true; + this.watchers.delete(clusterName); + } + } + if (removedServiceName) { + this.updateResourceNames(); + } + } + + getResourceNames(): string[] { + return Array.from(this.watchers.keys()); + } + + private validateResponse(message: Cluster__Output): boolean { + if (message.type !== 'EDS') { + return false; + } + if (!message.eds_cluster_config?.eds_config?.ads) { + return false; + } + if (message.lb_policy !== 'ROUND_ROBIN') { + return false; + } + if (message.lrs_server) { + if (!message.lrs_server.self) { + return false; + } + } + return true; + } + + /** + * Given a list of clusterNames (which may actually be the cluster name), + * for each watcher watching a name not on the list, call that watcher's + * onResourceDoesNotExist method. + * @param allClusterNames + */ + private handleMissingNames(allClusterNames: Set) { + for (const [clusterName, watcherList] of this.watchers.entries()) { + if (!allClusterNames.has(clusterName)) { + trace('Reporting CDS resource does not exist for clusterName ' + clusterName); + for (const watcher of watcherList) { + watcher.onResourceDoesNotExist(); + } + } + } + } + + handleResponses(responses: Cluster__Output[]): string | null { + for (const message of responses) { + if (!this.validateResponse(message)) { + trace('CDS validation failed for message ' + JSON.stringify(message)); + return 'CDS Error: Cluster validation failed'; + } + } + this.latestResponses = responses; + const allEdsServiceNames: Set = new Set(); + const allClusterNames: Set = new Set(); + for (const message of responses) { + allClusterNames.add(message.name); + const edsServiceName = message.eds_cluster_config?.service_name ?? ''; + allEdsServiceNames.add( + edsServiceName === '' ? message.name : edsServiceName + ); + const watchers = this.watchers.get(message.name) ?? []; + for (const watcher of watchers) { + watcher.onValidUpdate(message); + } + } + trace('Received CDS updates for cluster names ' + Array.from(allClusterNames)); + this.handleMissingNames(allClusterNames); + this.edsState.handleMissingNames(allEdsServiceNames); + return null; + } + + reportStreamError(status: StatusObject): void { + for (const watcherList of this.watchers.values()) { + for (const watcher of watcherList) { + watcher.onTransientError(status); + } + } + } +} \ No newline at end of file diff --git a/packages/grpc-js-xds/src/xds-stream-state/eds-state.ts b/packages/grpc-js-xds/src/xds-stream-state/eds-state.ts new file mode 100644 index 00000000..c9beef29 --- /dev/null +++ b/packages/grpc-js-xds/src/xds-stream-state/eds-state.ts @@ -0,0 +1,174 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { experimental, logVerbosity, StatusObject } from "@grpc/grpc-js"; +import { isIPv4, isIPv6 } from "net"; +import { ClusterLoadAssignment__Output } from "../generated/envoy/api/v2/ClusterLoadAssignment"; +import { Watcher, XdsStreamState } from "./xds-stream-state"; + +const TRACER_NAME = 'xds_client'; + +function trace(text: string): void { + experimental.trace(logVerbosity.DEBUG, TRACER_NAME, text); +} + +export class EdsState implements XdsStreamState { + public versionInfo = ''; + public nonce = ''; + + private watchers: Map< + string, + Watcher[] + > = new Map[]>(); + + private latestResponses: ClusterLoadAssignment__Output[] = []; + + constructor(private updateResourceNames: () => void) {} + + /** + * Add the watcher to the watcher list. Returns true if the list of resource + * names has changed, and false otherwise. + * @param edsServiceName + * @param watcher + */ + addWatcher( + edsServiceName: string, + watcher: Watcher + ): void { + let watchersEntry = this.watchers.get(edsServiceName); + let addedServiceName = false; + if (watchersEntry === undefined) { + addedServiceName = true; + watchersEntry = []; + this.watchers.set(edsServiceName, watchersEntry); + } + trace('Adding EDS watcher (' + watchersEntry.length + ' ->' + (watchersEntry.length + 1) + ') for edsServiceName ' + edsServiceName); + watchersEntry.push(watcher); + + /* If we have already received an update for the requested edsServiceName, + * immediately pass that update along to the watcher */ + for (const message of this.latestResponses) { + if (message.cluster_name === edsServiceName) { + /* These updates normally occur asynchronously, so we ensure that + * the same happens here */ + process.nextTick(() => { + trace('Reporting existing EDS update for new watcher for edsServiceName ' + edsServiceName); + watcher.onValidUpdate(message); + }); + } + } + if (addedServiceName) { + this.updateResourceNames(); + } + } + + removeWatcher( + edsServiceName: string, + watcher: Watcher + ): void { + trace('Removing EDS watcher for edsServiceName ' + edsServiceName); + const watchersEntry = this.watchers.get(edsServiceName); + let removedServiceName = false; + if (watchersEntry !== undefined) { + const entryIndex = watchersEntry.indexOf(watcher); + if (entryIndex >= 0) { + trace('Removed EDS watcher (' + watchersEntry.length + ' -> ' + (watchersEntry.length - 1) + ') for edsServiceName ' + edsServiceName); + watchersEntry.splice(entryIndex, 1); + } + if (watchersEntry.length === 0) { + removedServiceName = true; + this.watchers.delete(edsServiceName); + } + } + if (removedServiceName) { + this.updateResourceNames(); + } + } + + getResourceNames(): string[] { + return Array.from(this.watchers.keys()); + } + + /** + * Validate the ClusterLoadAssignment object by these rules: + * https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md#clusterloadassignment-proto + * @param message + */ + private validateResponse(message: ClusterLoadAssignment__Output) { + for (const endpoint of message.endpoints) { + for (const lb of endpoint.lb_endpoints) { + const socketAddress = lb.endpoint?.address?.socket_address; + if (!socketAddress) { + return false; + } + if (socketAddress.port_specifier !== 'port_value') { + return false; + } + if (!(isIPv4(socketAddress.address) || isIPv6(socketAddress.address))) { + return false; + } + } + } + return true; + } + + /** + * Given a list of edsServiceNames (which may actually be the cluster name), + * for each watcher watching a name not on the list, call that watcher's + * onResourceDoesNotExist method. + * @param allClusterNames + */ + handleMissingNames(allEdsServiceNames: Set) { + for (const [edsServiceName, watcherList] of this.watchers.entries()) { + if (!allEdsServiceNames.has(edsServiceName)) { + trace('Reporting EDS resource does not exist for edsServiceName ' + edsServiceName); + for (const watcher of watcherList) { + watcher.onResourceDoesNotExist(); + } + } + } + } + + handleResponses(responses: ClusterLoadAssignment__Output[]) { + for (const message of responses) { + if (!this.validateResponse(message)) { + trace('EDS validation failed for message ' + JSON.stringify(message)); + return 'EDS Error: ClusterLoadAssignment validation failed'; + } + } + this.latestResponses = responses; + const allClusterNames: Set = new Set(); + for (const message of responses) { + allClusterNames.add(message.cluster_name); + const watchers = this.watchers.get(message.cluster_name) ?? []; + for (const watcher of watchers) { + watcher.onValidUpdate(message); + } + } + trace('Received EDS updates for cluster names ' + Array.from(allClusterNames)); + this.handleMissingNames(allClusterNames); + return null; + } + + reportStreamError(status: StatusObject): void { + for (const watcherList of this.watchers.values()) { + for (const watcher of watcherList) { + watcher.onTransientError(status); + } + } + } +} \ No newline at end of file diff --git a/packages/grpc-js-xds/src/xds-stream-state/lds-state.ts b/packages/grpc-js-xds/src/xds-stream-state/lds-state.ts new file mode 100644 index 00000000..c5db3bfa --- /dev/null +++ b/packages/grpc-js-xds/src/xds-stream-state/lds-state.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import * as protoLoader from '@grpc/proto-loader'; +import { experimental, logVerbosity, StatusObject } from "@grpc/grpc-js"; +import { Listener__Output } from "../generated/envoy/api/v2/Listener"; +import { RdsState } from "./rds-state"; +import { XdsStreamState } from "./xds-stream-state"; +import { HttpConnectionManager__Output } from '../generated/envoy/config/filter/network/http_connection_manager/v2/HttpConnectionManager'; + +const TRACER_NAME = 'xds_client'; + +function trace(text: string): void { + experimental.trace(logVerbosity.DEBUG, TRACER_NAME, text); +} + +const HTTP_CONNECTION_MANGER_TYPE_URL = + 'type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager'; + +export class LdsState implements XdsStreamState { + versionInfo = ''; + nonce = ''; + + constructor(private targetName: string, private rdsState: RdsState) {} + + getResourceNames(): string[] { + return [this.targetName]; + } + + private validateResponse(message: Listener__Output): boolean { + if ( + !( + message.api_listener?.api_listener && + protoLoader.isAnyExtension(message.api_listener.api_listener) && + message.api_listener?.api_listener['@type'] === + HTTP_CONNECTION_MANGER_TYPE_URL + ) + ) { + return false; + } + const httpConnectionManager = message.api_listener + ?.api_listener as protoLoader.AnyExtension & + HttpConnectionManager__Output; + switch (httpConnectionManager.route_specifier) { + case 'rds': + return !!httpConnectionManager.rds?.config_source?.ads; + case 'route_config': + return true; + } + return false; + } + + handleResponses(responses: Listener__Output[]): string | null { + trace('Received LDS update with names ' + responses.map(message => message.name)); + for (const message of responses) { + if (message.name === this.targetName) { + if (this.validateResponse(message)) { + // The validation step ensures that this is correct + const httpConnectionManager = message.api_listener! + .api_listener as protoLoader.AnyExtension & + HttpConnectionManager__Output; + switch (httpConnectionManager.route_specifier) { + case 'rds': + trace('Received LDS update with RDS route config name ' + httpConnectionManager.rds!.route_config_name); + this.rdsState.setRouteConfigName( + httpConnectionManager.rds!.route_config_name + ); + break; + case 'route_config': + trace('Received LDS update with route configuration'); + this.rdsState.setRouteConfigName(null); + this.rdsState.handleSingleMessage( + httpConnectionManager.route_config! + ); + break; + default: + // The validation rules should prevent this + } + } else { + trace('LRS validation error for message ' + JSON.stringify(message)); + return 'LRS Error: Listener validation failed'; + } + } + } + return null; + } + + reportStreamError(status: StatusObject): void { + // Nothing to do here + } +} \ No newline at end of file diff --git a/packages/grpc-js-xds/src/xds-stream-state/rds-state.ts b/packages/grpc-js-xds/src/xds-stream-state/rds-state.ts new file mode 100644 index 00000000..18268587 --- /dev/null +++ b/packages/grpc-js-xds/src/xds-stream-state/rds-state.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { experimental, logVerbosity, StatusObject } from "@grpc/grpc-js"; +import { RouteConfiguration__Output } from "../generated/envoy/api/v2/RouteConfiguration"; +import { CdsLoadBalancingConfig } from "../load-balancer-cds"; +import { Watcher, XdsStreamState } from "./xds-stream-state"; +import ServiceConfig = experimental.ServiceConfig; + +const TRACER_NAME = 'xds_client'; + +function trace(text: string): void { + experimental.trace(logVerbosity.DEBUG, TRACER_NAME, text); +} + +export class RdsState implements XdsStreamState { + versionInfo = ''; + nonce = ''; + + private routeConfigName: string | null = null; + + constructor( + private targetName: string, + private watcher: Watcher, + private updateResouceNames: () => void + ) {} + + getResourceNames(): string[] { + return this.routeConfigName ? [this.routeConfigName] : []; + } + + handleSingleMessage(message: RouteConfiguration__Output) { + for (const virtualHost of message.virtual_hosts) { + if (virtualHost.domains.indexOf(this.targetName) >= 0) { + const route = virtualHost.routes[virtualHost.routes.length - 1]; + if (route.match?.prefix === '' && route.route?.cluster) { + trace('Reporting RDS update for host ' + this.targetName + ' with cluster ' + route.route.cluster); + this.watcher.onValidUpdate({ + methodConfig: [], + loadBalancingConfig: [ + new CdsLoadBalancingConfig(route.route.cluster) + ], + }); + return; + } else { + trace('Discarded matching route with prefix ' + route.match?.prefix + ' and cluster ' + route.route?.cluster); + } + } + } + trace('Reporting RDS resource does not exist from domain lists ' + message.virtual_hosts.map(virtualHost => virtualHost.domains)); + /* If none of the routes match the one we are looking for, bubble up an + * error. */ + this.watcher.onResourceDoesNotExist(); + } + + handleResponses(responses: RouteConfiguration__Output[]): string | null { + trace('Received RDS response with route config names ' + responses.map(message => message.name)); + if (this.routeConfigName !== null) { + for (const message of responses) { + if (message.name === this.routeConfigName) { + this.handleSingleMessage(message); + return null; + } + } + } + return null; + } + + setRouteConfigName(name: string | null) { + const oldName = this.routeConfigName; + this.routeConfigName = name; + if (name !== oldName) { + this.updateResouceNames(); + } + } + + reportStreamError(status: StatusObject): void { + this.watcher.onTransientError(status); + } +} \ No newline at end of file diff --git a/packages/grpc-js-xds/src/xds-stream-state/xds-stream-state.ts b/packages/grpc-js-xds/src/xds-stream-state/xds-stream-state.ts new file mode 100644 index 00000000..83db1781 --- /dev/null +++ b/packages/grpc-js-xds/src/xds-stream-state/xds-stream-state.ts @@ -0,0 +1,38 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { StatusObject } from "@grpc/grpc-js"; + +export interface Watcher { + onValidUpdate(update: UpdateType): void; + onTransientError(error: StatusObject): void; + onResourceDoesNotExist(): void; +} + +export interface XdsStreamState { + versionInfo: string; + nonce: string; + getResourceNames(): string[]; + /** + * Returns a string containing the error details if the message should be nacked, + * or null if it should be acked. + * @param responses + */ + handleResponses(responses: ResponseType[]): string | null; + + reportStreamError(status: StatusObject): void; +} \ No newline at end of file From 60eb600410fd58a8087e870c3dec988e1cccc46e Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Thu, 25 Feb 2021 10:02:44 -0800 Subject: [PATCH 23/46] move createGoogleDefaultCredentials from grpc-js to grpc-js-xds --- packages/grpc-js-xds/package.json | 3 ++- .../src/google-default-credentials.ts | 27 +++++++++++++++++++ packages/grpc-js-xds/src/xds-client.ts | 2 +- packages/grpc-js/package.json | 1 - packages/grpc-js/src/channel-credentials.ts | 11 -------- packages/grpc-js/src/experimental.ts | 1 - 6 files changed, 30 insertions(+), 15 deletions(-) create mode 100644 packages/grpc-js-xds/src/google-default-credentials.ts diff --git a/packages/grpc-js-xds/package.json b/packages/grpc-js-xds/package.json index f2b32a77..a26e03ea 100644 --- a/packages/grpc-js-xds/package.json +++ b/packages/grpc-js-xds/package.json @@ -42,7 +42,8 @@ "yargs": "^15.4.1" }, "dependencies": { - "@grpc/proto-loader": "^0.6.0-pre14" + "@grpc/proto-loader": "^0.6.0-pre14", + "google-auth-library": "^7.0.2" }, "peerDependencies": { "@grpc/grpc-js": "~1.2.2" diff --git a/packages/grpc-js-xds/src/google-default-credentials.ts b/packages/grpc-js-xds/src/google-default-credentials.ts new file mode 100644 index 00000000..ec765ee4 --- /dev/null +++ b/packages/grpc-js-xds/src/google-default-credentials.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import { GoogleAuth } from 'google-auth-library'; +import { ChannelCredentials, CallCredentials } from '@grpc/grpc-js'; + +export function createGoogleDefaultCredentials(): ChannelCredentials { + const sslCreds = ChannelCredentials.createSsl(); + const googleAuthCreds = CallCredentials.createFromGoogleCredential( + new GoogleAuth() + ); + return sslCreds.compose(googleAuthCreds); +} \ No newline at end of file diff --git a/packages/grpc-js-xds/src/xds-client.ts b/packages/grpc-js-xds/src/xds-client.ts index ae67960d..bedafda9 100644 --- a/packages/grpc-js-xds/src/xds-client.ts +++ b/packages/grpc-js-xds/src/xds-client.ts @@ -48,7 +48,7 @@ import { RouteConfiguration__Output } from './generated/envoy/api/v2/RouteConfig import { Any__Output } from './generated/google/protobuf/Any'; import BackoffTimeout = experimental.BackoffTimeout; import ServiceConfig = experimental.ServiceConfig; -import createGoogleDefaultCredentials = experimental.createGoogleDefaultCredentials; +import { createGoogleDefaultCredentials } from './google-default-credentials'; import { CdsLoadBalancingConfig } from './load-balancer-cds'; const TRACER_NAME = 'xds_client'; diff --git a/packages/grpc-js/package.json b/packages/grpc-js/package.json index 01a9d418..5d8360e6 100644 --- a/packages/grpc-js/package.json +++ b/packages/grpc-js/package.json @@ -58,7 +58,6 @@ }, "dependencies": { "@types/node": "^12.12.47", - "google-auth-library": "^6.1.1", "semver": "^6.2.0" }, "files": [ diff --git a/packages/grpc-js/src/channel-credentials.ts b/packages/grpc-js/src/channel-credentials.ts index aadf638b..675e9162 100644 --- a/packages/grpc-js/src/channel-credentials.ts +++ b/packages/grpc-js/src/channel-credentials.ts @@ -19,7 +19,6 @@ import { ConnectionOptions, createSecureContext, PeerCertificate } from 'tls'; import { CallCredentials } from './call-credentials'; import { CIPHER_SUITES, getDefaultRootsData } from './tls-helpers'; -import { GoogleAuth as GoogleAuthType } from 'google-auth-library'; // eslint-disable-next-line @typescript-eslint/no-explicit-any function verifyIsBufferOrNull(obj: any, friendlyName: string): void { @@ -279,13 +278,3 @@ class ComposedChannelCredentialsImpl extends ChannelCredentials { } } } - -export function createGoogleDefaultCredentials(): ChannelCredentials { - const GoogleAuth = require('google-auth-library') - .GoogleAuth as typeof GoogleAuthType; - const sslCreds = ChannelCredentials.createSsl(); - const googleAuthCreds = CallCredentials.createFromGoogleCredential( - new GoogleAuth() - ); - return sslCreds.compose(googleAuthCreds); -} diff --git a/packages/grpc-js/src/experimental.ts b/packages/grpc-js/src/experimental.ts index b88f124a..c27f9810 100644 --- a/packages/grpc-js/src/experimental.ts +++ b/packages/grpc-js/src/experimental.ts @@ -2,7 +2,6 @@ export { trace } from './logging'; export { Resolver, ResolverListener, registerResolver } from './resolver'; export { GrpcUri, uriToString } from './uri-parser'; export { ServiceConfig } from './service-config'; -export { createGoogleDefaultCredentials } from './channel-credentials'; export { BackoffTimeout } from './backoff-timeout'; export { LoadBalancer, LoadBalancingConfig, ChannelControlHelper, registerLoadBalancerType, getFirstUsableConfig, validateLoadBalancingConfig } from './load-balancer'; export { SubchannelAddress, subchannelAddressToString } from './subchannel'; From 131b604f2c01dffa80e34ae92df4174a8724311f Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Wed, 24 Feb 2021 14:25:41 -0800 Subject: [PATCH 24/46] Add routing and traffic splitting functionality --- packages/grpc-js-xds/package.json | 3 +- packages/grpc-js-xds/src/environment.ts | 18 + packages/grpc-js-xds/src/load-balancer-cds.ts | 10 +- packages/grpc-js-xds/src/load-balancer-eds.ts | 15 +- packages/grpc-js-xds/src/resolver-xds.ts | 444 ++++++++++++++- packages/grpc-js-xds/src/xds-client.ts | 532 ++---------------- .../src/xds-stream-state/lds-state.ts | 118 ++-- .../src/xds-stream-state/rds-state.ts | 183 ++++-- packages/grpc-js/src/experimental.ts | 2 +- 9 files changed, 723 insertions(+), 602 deletions(-) create mode 100644 packages/grpc-js-xds/src/environment.ts diff --git a/packages/grpc-js-xds/package.json b/packages/grpc-js-xds/package.json index f2b32a77..eff02b68 100644 --- a/packages/grpc-js-xds/package.json +++ b/packages/grpc-js-xds/package.json @@ -42,7 +42,8 @@ "yargs": "^15.4.1" }, "dependencies": { - "@grpc/proto-loader": "^0.6.0-pre14" + "@grpc/proto-loader": "^0.6.0-pre14", + "re2-wasm": "^1.0.1" }, "peerDependencies": { "@grpc/grpc-js": "~1.2.2" diff --git a/packages/grpc-js-xds/src/environment.ts b/packages/grpc-js-xds/src/environment.ts new file mode 100644 index 00000000..67fc531e --- /dev/null +++ b/packages/grpc-js-xds/src/environment.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export const GRPC_XDS_EXPERIMENTAL_ROUTING = (process.env.GRPC_XDS_EXPERIMENTAL_ROUTING === 'true'); \ No newline at end of file diff --git a/packages/grpc-js-xds/src/load-balancer-cds.ts b/packages/grpc-js-xds/src/load-balancer-cds.ts index 452b1304..9a372588 100644 --- a/packages/grpc-js-xds/src/load-balancer-cds.ts +++ b/packages/grpc-js-xds/src/load-balancer-cds.ts @@ -16,7 +16,7 @@ */ import { connectivityState, status, Metadata, logVerbosity, experimental } from '@grpc/grpc-js'; -import { XdsClient } from './xds-client'; +import { getSingletonXdsClient, XdsClient } from './xds-client'; import { Cluster__Output } from './generated/envoy/api/v2/Cluster'; import SubchannelAddress = experimental.SubchannelAddress; import UnavailablePicker = experimental.UnavailablePicker; @@ -66,7 +66,6 @@ export class CdsLoadBalancingConfig implements LoadBalancingConfig { export class CdsLoadBalancer implements LoadBalancer { private childBalancer: ChildLoadBalancerHandler; - private xdsClient: XdsClient | null = null; private watcher: Watcher; private isWatcherActive = false; @@ -127,7 +126,6 @@ export class CdsLoadBalancer implements LoadBalancer { return; } trace('Received update with config ' + JSON.stringify(lbConfig, undefined, 2)); - this.xdsClient = attributes.xdsClient; this.latestAttributes = attributes; /* If the cluster is changing, disable the old watcher before adding the new @@ -137,7 +135,7 @@ export class CdsLoadBalancer implements LoadBalancer { this.latestConfig?.getCluster() !== lbConfig.getCluster() ) { trace('Removing old cluster watcher for cluster name ' + this.latestConfig!.getCluster()); - this.xdsClient.removeClusterWatcher( + getSingletonXdsClient().removeClusterWatcher( this.latestConfig!.getCluster(), this.watcher ); @@ -153,7 +151,7 @@ export class CdsLoadBalancer implements LoadBalancer { if (!this.isWatcherActive) { trace('Adding new cluster watcher for cluster name ' + lbConfig.getCluster()); - this.xdsClient.addClusterWatcher(lbConfig.getCluster(), this.watcher); + getSingletonXdsClient().addClusterWatcher(lbConfig.getCluster(), this.watcher); this.isWatcherActive = true; } } @@ -167,7 +165,7 @@ export class CdsLoadBalancer implements LoadBalancer { trace('Destroying load balancer with cluster name ' + this.latestConfig?.getCluster()); this.childBalancer.destroy(); if (this.isWatcherActive) { - this.xdsClient?.removeClusterWatcher( + getSingletonXdsClient().removeClusterWatcher( this.latestConfig!.getCluster(), this.watcher ); diff --git a/packages/grpc-js-xds/src/load-balancer-eds.ts b/packages/grpc-js-xds/src/load-balancer-eds.ts index 8919f317..35da2a46 100644 --- a/packages/grpc-js-xds/src/load-balancer-eds.ts +++ b/packages/grpc-js-xds/src/load-balancer-eds.ts @@ -16,7 +16,7 @@ */ import { connectivityState as ConnectivityState, status as Status, Metadata, logVerbosity as LogVerbosity, experimental } from '@grpc/grpc-js'; -import { XdsClient, Watcher, XdsClusterDropStats } from './xds-client'; +import { getSingletonXdsClient, XdsClient, XdsClusterDropStats } from './xds-client'; import { ClusterLoadAssignment__Output } from './generated/envoy/api/v2/ClusterLoadAssignment'; import { Locality__Output } from './generated/envoy/api/v2/core/Locality'; import { LocalitySubchannelAddress, PriorityChild, PriorityLoadBalancingConfig } from './load-balancer-priority'; @@ -33,6 +33,7 @@ import PickResultType = experimental.PickResultType; import { validateLoadBalancingConfig } from '@grpc/grpc-js/build/src/experimental'; import { WeightedTarget, WeightedTargetLoadBalancingConfig } from './load-balancer-weighted-target'; import { LrsLoadBalancingConfig } from './load-balancer-lrs'; +import { Watcher } from './xds-stream-state/xds-stream-state'; const TRACER_NAME = 'eds_balancer'; @@ -122,11 +123,10 @@ export class EdsLoadBalancer implements LoadBalancer { * requests. */ private childBalancer: ChildLoadBalancerHandler; - private xdsClient: XdsClient | null = null; private edsServiceName: string | null = null; private watcher: Watcher; /** - * Indicates whether the watcher has already been passed to this.xdsClient + * Indicates whether the watcher has already been passed to the xdsClient * and is getting updates. */ private isWatcherActive = false; @@ -434,14 +434,13 @@ export class EdsLoadBalancer implements LoadBalancer { trace('Received update with config: ' + JSON.stringify(lbConfig, undefined, 2)); this.lastestConfig = lbConfig; this.latestAttributes = attributes; - this.xdsClient = attributes.xdsClient; const newEdsServiceName = lbConfig.getEdsServiceName() ?? lbConfig.getCluster(); /* If the name is changing, disable the old watcher before adding the new * one */ if (this.isWatcherActive && this.edsServiceName !== newEdsServiceName) { trace('Removing old endpoint watcher for edsServiceName ' + this.edsServiceName) - this.xdsClient.removeEndpointWatcher(this.edsServiceName!, this.watcher); + getSingletonXdsClient().removeEndpointWatcher(this.edsServiceName!, this.watcher); /* Setting isWatcherActive to false here lets us have one code path for * calling addEndpointWatcher */ this.isWatcherActive = false; @@ -454,12 +453,12 @@ export class EdsLoadBalancer implements LoadBalancer { if (!this.isWatcherActive) { trace('Adding new endpoint watcher for edsServiceName ' + this.edsServiceName); - this.xdsClient.addEndpointWatcher(this.edsServiceName, this.watcher); + getSingletonXdsClient().addEndpointWatcher(this.edsServiceName, this.watcher); this.isWatcherActive = true; } if (lbConfig.getLrsLoadReportingServerName()) { - this.clusterDropStats = this.xdsClient.addClusterDropStats( + this.clusterDropStats = getSingletonXdsClient().addClusterDropStats( lbConfig.getLrsLoadReportingServerName()!, lbConfig.getCluster(), lbConfig.getEdsServiceName() ?? '' @@ -480,7 +479,7 @@ export class EdsLoadBalancer implements LoadBalancer { destroy(): void { trace('Destroying load balancer with edsServiceName ' + this.edsServiceName); if (this.edsServiceName) { - this.xdsClient?.removeEndpointWatcher(this.edsServiceName, this.watcher); + getSingletonXdsClient().removeEndpointWatcher(this.edsServiceName, this.watcher); } this.childBalancer.destroy(); } diff --git a/packages/grpc-js-xds/src/resolver-xds.ts b/packages/grpc-js-xds/src/resolver-xds.ts index 1588f3b1..b3c78ada 100644 --- a/packages/grpc-js-xds/src/resolver-xds.ts +++ b/packages/grpc-js-xds/src/resolver-xds.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { XdsClient } from './xds-client'; +import * as protoLoader from '@grpc/proto-loader'; + +import { RE2 } from 're2-wasm'; + +import { getSingletonXdsClient, XdsClient } from './xds-client'; import { StatusObject, status, logVerbosity, Metadata, experimental, ChannelOptions } from '@grpc/grpc-js'; import Resolver = experimental.Resolver; import GrpcUri = experimental.GrpcUri; @@ -22,6 +26,17 @@ import ResolverListener = experimental.ResolverListener; import uriToString = experimental.uriToString; import ServiceConfig = experimental.ServiceConfig; import registerResolver = experimental.registerResolver; +import { Listener__Output } from './generated/envoy/api/v2/Listener'; +import { Watcher } from './xds-stream-state/xds-stream-state'; +import { RouteConfiguration__Output } from './generated/envoy/api/v2/RouteConfiguration'; +import { HttpConnectionManager__Output } from './generated/envoy/config/filter/network/http_connection_manager/v2/HttpConnectionManager'; +import { GRPC_XDS_EXPERIMENTAL_ROUTING } from './environment'; +import { CdsLoadBalancingConfig } from './load-balancer-cds'; +import { VirtualHost__Output } from './generated/envoy/api/v2/route/VirtualHost'; +import { RouteMatch__Output } from './generated/envoy/api/v2/route/RouteMatch'; +import { HeaderMatcher__Output } from './generated/envoy/api/v2/route/HeaderMatcher'; +import ConfigSelector = experimental.ConfigSelector; +import LoadBalancingConfig = experimental.LoadBalancingConfig; const TRACER_NAME = 'xds_resolver'; @@ -29,15 +44,404 @@ function trace(text: string): void { experimental.trace(logVerbosity.DEBUG, TRACER_NAME, text); } +// Better match type has smaller value. +enum MatchType { + EXACT_MATCH, + SUFFIX_MATCH, + PREFIX_MATCH, + UNIVERSE_MATCH, + INVALID_MATCH, +}; + +function domainPatternMatchType(domainPattern: string): MatchType { + if (domainPattern.length === 0) { + return MatchType.INVALID_MATCH; + } + if (domainPattern.indexOf('*') < 0) { + return MatchType.EXACT_MATCH; + } + if (domainPattern === '*') { + return MatchType.UNIVERSE_MATCH; + } + if (domainPattern.startsWith('*')) { + return MatchType.SUFFIX_MATCH; + } + if (domainPattern.endsWith('*')) { + return MatchType.PREFIX_MATCH; + } + return MatchType.INVALID_MATCH; +} + +function domainMatch(matchType: MatchType, domainPattern: string, expectedHostName: string) { + switch (matchType) { + case MatchType.EXACT_MATCH: + return expectedHostName === domainPattern; + case MatchType.SUFFIX_MATCH: + return expectedHostName.endsWith(domainPattern.substring(1)); + case MatchType.PREFIX_MATCH: + return expectedHostName.startsWith(domainPattern.substring(0, domainPattern.length - 1)); + case MatchType.UNIVERSE_MATCH: + return true; + case MatchType.INVALID_MATCH: + return false; + } +} + +function findVirtualHostForDomain(virutalHostList: VirtualHost__Output[], domain: string): VirtualHost__Output | null { + let targetVhost: VirtualHost__Output | null = null; + let bestMatchType: MatchType = MatchType.INVALID_MATCH; + let longestMatch = 0; + for (const virtualHost of virutalHostList) { + for (const domainPattern of virtualHost.domains) { + const matchType = domainPatternMatchType(domainPattern); + // If we already have a match of a better type, skip this one + if (matchType > bestMatchType) { + continue; + } + // If we already have a longer match of the same type, skip this one + if (matchType === bestMatchType && domainPattern.length <= longestMatch) { + continue; + } + if (domainMatch(matchType, domainPattern, domain)) { + targetVhost = virtualHost; + bestMatchType = matchType; + longestMatch = domainPattern.length; + } + if (bestMatchType === MatchType.EXACT_MATCH) { + break; + } + } + if (bestMatchType === MatchType.EXACT_MATCH) { + break; + } + } + return targetVhost; +} + +interface Matcher { + (methodName: string, metadata: Metadata): boolean; +} + +const numberRegex = new RE2(/^-?\d+$/u); + +function getPredicateForHeaderMatcher(headerMatch: HeaderMatcher__Output): Matcher { + let valueChecker: (value: string) => boolean; + switch (headerMatch.header_match_specifier) { + case 'exact_match': + valueChecker = value => value === headerMatch.exact_match; + break; + case 'safe_regex_match': + const regex = new RE2(`^${headerMatch.safe_regex_match}$`, 'u'); + valueChecker = value => regex.test(value); + break; + case 'range_match': + const start = BigInt(headerMatch.range_match!.start); + const end = BigInt(headerMatch.range_match!.end); + valueChecker = value => { + if (!numberRegex.test(value)) { + return false; + } + const numberValue = BigInt(value); + return start <= numberValue && numberValue < end; + } + break; + case 'present_match': + valueChecker = value => true; + break; + case 'prefix_match': + valueChecker = value => value.startsWith(headerMatch.prefix_match!); + break; + case 'suffix_match': + valueChecker = value => value.endsWith(headerMatch.suffix_match!); + break; + default: + // Should be prevented by validation rules + return (methodName, metadata) => false; + } + const headerMatcher: Matcher = (methodName, metadata) => { + if (headerMatch.name.endsWith('-bin')) { + return false; + } + let value: string; + if (headerMatch.name === 'content-type') { + value = 'application/grpc'; + } else { + const valueArray = metadata.get(headerMatch.name); + if (valueArray.length === 0) { + return false; + } else { + value = valueArray.join(','); + } + } + return valueChecker(value); + } + if (headerMatch.invert_match) { + return (methodName, metadata) => !headerMatcher(methodName, metadata); + } else { + return headerMatcher; + } +} + +const RUNTIME_FRACTION_DENOMINATOR_VALUES = { + HUNDRED: 100, + TEN_THOUSAND: 10_000, + MILLION: 1_000_000 +} + +function getPredicateForMatcher(routeMatch: RouteMatch__Output): Matcher { + let pathMatcher: Matcher; + switch (routeMatch.path_specifier) { + case 'prefix': + if (routeMatch.case_sensitive?.value === false) { + const prefix = routeMatch.prefix!.toLowerCase(); + pathMatcher = (methodName, metadata) => (methodName.toLowerCase().startsWith(prefix)); + } else { + const prefix = routeMatch.prefix!; + pathMatcher = (methodName, metadata) => (methodName.startsWith(prefix)); + } + break; + case 'path': + if (routeMatch.case_sensitive?.value === false) { + const path = routeMatch.path!.toLowerCase(); + pathMatcher = (methodName, metadata) => (methodName.toLowerCase() === path); + } else { + const path = routeMatch.path!; + pathMatcher = (methodName, metadata) => (methodName === path); + } + break; + case 'safe_regex': + const flags = routeMatch.case_sensitive?.value === false ? 'ui' : 'u'; + const regex = new RE2(`^${routeMatch.safe_regex!.regex!}$`, flags); + pathMatcher = (methodName, metadata) => (regex.test(methodName)); + break; + default: + // Should be prevented by validation rules + return (methodName, metadata) => false; + } + const headerMatchers: Matcher[] = routeMatch.headers.map(getPredicateForHeaderMatcher); + let runtimeFractionHandler: () => boolean; + if (!routeMatch.runtime_fraction?.default_value) { + runtimeFractionHandler = () => true; + } else { + const numerator = routeMatch.runtime_fraction.default_value.numerator; + const denominator = RUNTIME_FRACTION_DENOMINATOR_VALUES[routeMatch.runtime_fraction.default_value.denominator]; + runtimeFractionHandler = () => { + const randomNumber = Math.random() * denominator; + return randomNumber < numerator; + } + } + return (methodName, metadata) => pathMatcher(methodName, metadata) && headerMatchers.every(matcher => matcher(methodName, metadata)) && runtimeFractionHandler(); +} + class XdsResolver implements Resolver { private hasReportedSuccess = false; - private xdsClient: XdsClient | null = null; + + private ldsWatcher: Watcher; + private rdsWatcher: Watcher + private isLdsWatcherActive = false; + /** + * The latest route config name from an LDS response. The RDS watcher is + * actively watching that name if and only if this is not null. + */ + private latestRouteConfigName: string | null = null; + + private latestRouteConfig: RouteConfiguration__Output | null = null; + + private clusterRefcounts = new Map(); constructor( private target: GrpcUri, private listener: ResolverListener, private channelOptions: ChannelOptions - ) {} + ) { + this.ldsWatcher = { + onValidUpdate: (update: Listener__Output) => { + const httpConnectionManager = update.api_listener! + .api_listener as protoLoader.AnyExtension & + HttpConnectionManager__Output; + switch (httpConnectionManager.route_specifier) { + case 'rds': { + const routeConfigName = httpConnectionManager.rds!.route_config_name; + if (this.latestRouteConfigName !== routeConfigName) { + if (this.latestRouteConfigName !== null) { + getSingletonXdsClient().removeRouteWatcher(this.latestRouteConfigName, this.rdsWatcher); + } + getSingletonXdsClient().addRouteWatcher(httpConnectionManager.rds!.route_config_name, this.rdsWatcher); + this.latestRouteConfigName = routeConfigName; + } + break; + } + case 'route_config': + if (this.latestRouteConfigName) { + getSingletonXdsClient().removeRouteWatcher(this.latestRouteConfigName, this.rdsWatcher); + } + this.handleRouteConfig(httpConnectionManager.route_config!); + break; + default: + // This is prevented by the validation rules + } + }, + onTransientError: (error: StatusObject) => { + /* A transient error only needs to bubble up as a failure if we have + * not already provided a ServiceConfig for the upper layer to use */ + if (!this.hasReportedSuccess) { + trace('Resolution error for target ' + uriToString(this.target) + ' due to xDS client transient error ' + error.details); + this.reportResolutionError(error.details); + } + }, + onResourceDoesNotExist: () => { + trace('Resolution error for target ' + uriToString(this.target) + ': LDS resource does not exist'); + this.reportResolutionError(`Listener ${this.target} does not exist`); + } + }; + this.rdsWatcher = { + onValidUpdate: (update: RouteConfiguration__Output) => { + this.handleRouteConfig(update); + }, + onTransientError: (error: StatusObject) => { + /* A transient error only needs to bubble up as a failure if we have + * not already provided a ServiceConfig for the upper layer to use */ + if (!this.hasReportedSuccess) { + trace('Resolution error for target ' + uriToString(this.target) + ' due to xDS client transient error ' + error.details); + this.reportResolutionError(error.details); + } + }, + onResourceDoesNotExist: () => { + trace('Resolution error for target ' + uriToString(this.target) + ' and route config ' + this.latestRouteConfigName + ': RDS resource does not exist'); + this.reportResolutionError(`Route config ${this.latestRouteConfigName} does not exist`); + } + } + } + + private refCluster(clusterName: string) { + const refCount = this.clusterRefcounts.get(clusterName); + if (refCount) { + refCount.refCount += 1; + } + } + + private unrefCluster(clusterName: string) { + const refCount = this.clusterRefcounts.get(clusterName); + if (refCount) { + refCount.refCount -= 1; + if (!refCount.inLastConfig && refCount.refCount === 0) { + this.clusterRefcounts.delete(clusterName); + this.handleRouteConfig(this.latestRouteConfig!); + } + } + } + + private handleRouteConfig(routeConfig: RouteConfiguration__Output) { + this.latestRouteConfig = routeConfig; + if (GRPC_XDS_EXPERIMENTAL_ROUTING) { + const virtualHost = findVirtualHostForDomain(routeConfig.virtual_hosts, this.target.path); + if (virtualHost === null) { + this.reportResolutionError('No matching route found'); + return; + } + const allConfigClusters = new Set(); + const matchList: {matcher: Matcher, action: () => string}[] = []; + for (const route of virtualHost.routes) { + let routeAction: () => string; + switch (route.route!.cluster_specifier) { + case 'cluster_header': + continue; + case 'cluster':{ + const cluster = route.route!.cluster!; + allConfigClusters.add(cluster); + routeAction = () => cluster; + break; + } + case 'weighted_clusters': { + let lastNumerator = 0; + // clusterChoices is essentially the weighted choices represented as a CDF + const clusterChoices: {cluster: string, numerator: number}[] = []; + for (const clusterWeight of route.route!.weighted_clusters!.clusters) { + allConfigClusters.add(clusterWeight.name); + lastNumerator = lastNumerator + (clusterWeight.weight?.value ?? 0); + clusterChoices.push({cluster: clusterWeight.name, numerator: lastNumerator}); + } + routeAction = () => { + const randomNumber = Math.random() * (route.route!.weighted_clusters!.total_weight?.value ?? 100); + for (const choice of clusterChoices) { + if (randomNumber < choice.numerator) { + return choice.cluster; + } + } + // This should be prevented by the validation rules + return ''; + } + } + } + const routeMatcher = getPredicateForMatcher(route.match!); + matchList.push({matcher: routeMatcher, action: routeAction}); + } + // Mark clusters that are not in this config, and remove ones with no references + for (const [name, refCount] of Array.from(this.clusterRefcounts.entries())) { + if (!allConfigClusters.has(name)) { + refCount.inLastConfig = false; + if (refCount.refCount === 0) { + this.clusterRefcounts.delete(name); + } + } + } + for (const name of allConfigClusters) { + if (this.clusterRefcounts.has(name)) { + this.clusterRefcounts.get(name)!.inLastConfig = true; + } else { + this.clusterRefcounts.set(name, {inLastConfig: true, refCount: 0}); + } + } + const configSelector: ConfigSelector = (methodName, metadata) => { + for (const {matcher, action} of matchList) { + if (matcher(methodName, metadata)) { + const clusterName = action(); + this.refCluster(clusterName); + const onCommitted = () => { + this.unrefCluster(clusterName); + } + return { + methodConfig: {name: []}, + onCommitted: onCommitted, + pickInformation: {cluster: clusterName}, + status: status.OK + }; + } + } + return { + methodConfig: {name: []}, + // pickInformation won't be used here, but it's set because of some TypeScript weirdness + pickInformation: {cluster: ''}, + status: status.UNAVAILABLE + }; + }; + const clusterConfigMap = new Map(); + for (const clusterName of this.clusterRefcounts.keys()) { + clusterConfigMap.set(clusterName, {child_policy: [new CdsLoadBalancingConfig(clusterName)]}); + } + // TODO: Create xdsClusterManagerLoadBalancingConfig and report successful resolution + } else { + for (const virtualHost of routeConfig.virtual_hosts) { + if (virtualHost.domains.indexOf(this.target.path) >= 0) { + const route = virtualHost.routes[virtualHost.routes.length - 1]; + if (route.match?.prefix === '' && route.route?.cluster) { + trace('Reporting RDS update for host ' + uriToString(this.target) + ' with cluster ' + route.route.cluster); + this.listener.onSuccessfulResolution([], { + methodConfig: [], + loadBalancingConfig: [ + new CdsLoadBalancingConfig(route.route.cluster) + ], + }, null, null, {}); + this.hasReportedSuccess = true; + return; + } else { + trace('Discarded matching route with prefix ' + route.match?.prefix + ' and cluster ' + route.route?.cluster); + } + } + } + this.reportResolutionError('No matching route found'); + } + } private reportResolutionError(reason: string) { this.listener.onError({ @@ -51,38 +455,18 @@ class XdsResolver implements Resolver { updateResolution(): void { // Wait until updateResolution is called once to start the xDS requests - if (this.xdsClient === null) { + if (!this.isLdsWatcherActive) { trace('Starting resolution for target ' + uriToString(this.target)); - this.xdsClient = new XdsClient( - this.target.path, - { - onValidUpdate: (update: ServiceConfig) => { - trace('Resolved service config for target ' + uriToString(this.target) + ': ' + JSON.stringify(update)); - this.hasReportedSuccess = true; - this.listener.onSuccessfulResolution([], update, null, null, { - xdsClient: this.xdsClient, - }); - }, - onTransientError: (error: StatusObject) => { - /* A transient error only needs to bubble up as a failure if we have - * not already provided a ServiceConfig for the upper layer to use */ - if (!this.hasReportedSuccess) { - trace('Resolution error for target ' + uriToString(this.target) + ' due to xDS client transient error ' + error.details); - this.reportResolutionError(error.details); - } - }, - onResourceDoesNotExist: () => { - trace('Resolution error for target ' + uriToString(this.target) + ': resource does not exist'); - this.reportResolutionError("Resource does not exist"); - }, - }, - this.channelOptions - ); + getSingletonXdsClient().addListenerWatcher(this.target.path, this.ldsWatcher); + this.isLdsWatcherActive = true; } } destroy() { - this.xdsClient?.shutdown(); + getSingletonXdsClient().removeListenerWatcher(this.target.path, this.ldsWatcher); + if (this.latestRouteConfigName) { + getSingletonXdsClient().removeRouteWatcher(this.latestRouteConfigName, this.rdsWatcher); + } } static getDefaultAuthority(target: GrpcUri) { diff --git a/packages/grpc-js-xds/src/xds-client.ts b/packages/grpc-js-xds/src/xds-client.ts index ae67960d..28994943 100644 --- a/packages/grpc-js-xds/src/xds-client.ts +++ b/packages/grpc-js-xds/src/xds-client.ts @@ -50,6 +50,11 @@ import BackoffTimeout = experimental.BackoffTimeout; import ServiceConfig = experimental.ServiceConfig; import createGoogleDefaultCredentials = experimental.createGoogleDefaultCredentials; import { CdsLoadBalancingConfig } from './load-balancer-cds'; +import { EdsState } from './xds-stream-state/eds-state'; +import { CdsState } from './xds-stream-state/cds-state'; +import { RdsState } from './xds-stream-state/rds-state'; +import { LdsState } from './xds-stream-state/lds-state'; +import { Watcher } from './xds-stream-state/xds-stream-state'; const TRACER_NAME = 'xds_client'; @@ -131,12 +136,6 @@ function localityEqual( ); } -export interface Watcher { - onValidUpdate(update: UpdateType): void; - onTransientError(error: StatusObject): void; - onResourceDoesNotExist(): void; -} - export interface XdsClusterDropStats { addCallDropped(category: string): void; } @@ -219,450 +218,6 @@ class ClusterLoadReportMap { } } -interface XdsStreamState { - versionInfo: string; - nonce: string; - getResourceNames(): string[]; - /** - * Returns a string containing the error details if the message should be nacked, - * or null if it should be acked. - * @param responses - */ - handleResponses(responses: ResponseType[]): string | null; - - reportStreamError(status: StatusObject): void; -} - -class EdsState implements XdsStreamState { - public versionInfo = ''; - public nonce = ''; - - private watchers: Map< - string, - Watcher[] - > = new Map[]>(); - - private latestResponses: ClusterLoadAssignment__Output[] = []; - - constructor(private updateResourceNames: () => void) {} - - /** - * Add the watcher to the watcher list. Returns true if the list of resource - * names has changed, and false otherwise. - * @param edsServiceName - * @param watcher - */ - addWatcher( - edsServiceName: string, - watcher: Watcher - ): void { - let watchersEntry = this.watchers.get(edsServiceName); - let addedServiceName = false; - if (watchersEntry === undefined) { - addedServiceName = true; - watchersEntry = []; - this.watchers.set(edsServiceName, watchersEntry); - } - trace('Adding EDS watcher (' + watchersEntry.length + ' ->' + (watchersEntry.length + 1) + ') for edsServiceName ' + edsServiceName); - watchersEntry.push(watcher); - - /* If we have already received an update for the requested edsServiceName, - * immediately pass that update along to the watcher */ - for (const message of this.latestResponses) { - if (message.cluster_name === edsServiceName) { - /* These updates normally occur asynchronously, so we ensure that - * the same happens here */ - process.nextTick(() => { - trace('Reporting existing EDS update for new watcher for edsServiceName ' + edsServiceName); - watcher.onValidUpdate(message); - }); - } - } - if (addedServiceName) { - this.updateResourceNames(); - } - } - - removeWatcher( - edsServiceName: string, - watcher: Watcher - ): void { - trace('Removing EDS watcher for edsServiceName ' + edsServiceName); - const watchersEntry = this.watchers.get(edsServiceName); - let removedServiceName = false; - if (watchersEntry !== undefined) { - const entryIndex = watchersEntry.indexOf(watcher); - if (entryIndex >= 0) { - trace('Removed EDS watcher (' + watchersEntry.length + ' -> ' + (watchersEntry.length - 1) + ') for edsServiceName ' + edsServiceName); - watchersEntry.splice(entryIndex, 1); - } - if (watchersEntry.length === 0) { - removedServiceName = true; - this.watchers.delete(edsServiceName); - } - } - if (removedServiceName) { - this.updateResourceNames(); - } - } - - getResourceNames(): string[] { - return Array.from(this.watchers.keys()); - } - - /** - * Validate the ClusterLoadAssignment object by these rules: - * https://github.com/grpc/proposal/blob/master/A27-xds-global-load-balancing.md#clusterloadassignment-proto - * @param message - */ - private validateResponse(message: ClusterLoadAssignment__Output) { - for (const endpoint of message.endpoints) { - for (const lb of endpoint.lb_endpoints) { - const socketAddress = lb.endpoint?.address?.socket_address; - if (!socketAddress) { - return false; - } - if (socketAddress.port_specifier !== 'port_value') { - return false; - } - if (!(isIPv4(socketAddress.address) || isIPv6(socketAddress.address))) { - return false; - } - } - } - return true; - } - - /** - * Given a list of edsServiceNames (which may actually be the cluster name), - * for each watcher watching a name not on the list, call that watcher's - * onResourceDoesNotExist method. - * @param allClusterNames - */ - handleMissingNames(allEdsServiceNames: Set) { - for (const [edsServiceName, watcherList] of this.watchers.entries()) { - if (!allEdsServiceNames.has(edsServiceName)) { - trace('Reporting EDS resource does not exist for edsServiceName ' + edsServiceName); - for (const watcher of watcherList) { - watcher.onResourceDoesNotExist(); - } - } - } - } - - handleResponses(responses: ClusterLoadAssignment__Output[]) { - for (const message of responses) { - if (!this.validateResponse(message)) { - trace('EDS validation failed for message ' + JSON.stringify(message)); - return 'EDS Error: ClusterLoadAssignment validation failed'; - } - } - this.latestResponses = responses; - const allClusterNames: Set = new Set(); - for (const message of responses) { - allClusterNames.add(message.cluster_name); - const watchers = this.watchers.get(message.cluster_name) ?? []; - for (const watcher of watchers) { - watcher.onValidUpdate(message); - } - } - trace('Received EDS updates for cluster names ' + Array.from(allClusterNames)); - this.handleMissingNames(allClusterNames); - return null; - } - - reportStreamError(status: StatusObject): void { - for (const watcherList of this.watchers.values()) { - for (const watcher of watcherList) { - watcher.onTransientError(status); - } - } - } -} - -class CdsState implements XdsStreamState { - versionInfo = ''; - nonce = ''; - - private watchers: Map[]> = new Map< - string, - Watcher[] - >(); - - private latestResponses: Cluster__Output[] = []; - - constructor( - private edsState: EdsState, - private updateResourceNames: () => void - ) {} - - /** - * Add the watcher to the watcher list. Returns true if the list of resource - * names has changed, and false otherwise. - * @param clusterName - * @param watcher - */ - addWatcher(clusterName: string, watcher: Watcher): void { - trace('Adding CDS watcher for clusterName ' + clusterName); - let watchersEntry = this.watchers.get(clusterName); - let addedServiceName = false; - if (watchersEntry === undefined) { - addedServiceName = true; - watchersEntry = []; - this.watchers.set(clusterName, watchersEntry); - } - watchersEntry.push(watcher); - - /* If we have already received an update for the requested edsServiceName, - * immediately pass that update along to the watcher */ - for (const message of this.latestResponses) { - if (message.name === clusterName) { - /* These updates normally occur asynchronously, so we ensure that - * the same happens here */ - process.nextTick(() => { - trace('Reporting existing CDS update for new watcher for clusterName ' + clusterName); - watcher.onValidUpdate(message); - }); - } - } - if (addedServiceName) { - this.updateResourceNames(); - } - } - - removeWatcher(clusterName: string, watcher: Watcher): void { - trace('Removing CDS watcher for clusterName ' + clusterName); - const watchersEntry = this.watchers.get(clusterName); - let removedServiceName = false; - if (watchersEntry !== undefined) { - const entryIndex = watchersEntry.indexOf(watcher); - if (entryIndex >= 0) { - watchersEntry.splice(entryIndex, 1); - } - if (watchersEntry.length === 0) { - removedServiceName = true; - this.watchers.delete(clusterName); - } - } - if (removedServiceName) { - this.updateResourceNames(); - } - } - - getResourceNames(): string[] { - return Array.from(this.watchers.keys()); - } - - private validateResponse(message: Cluster__Output): boolean { - if (message.type !== 'EDS') { - return false; - } - if (!message.eds_cluster_config?.eds_config?.ads) { - return false; - } - if (message.lb_policy !== 'ROUND_ROBIN') { - return false; - } - if (message.lrs_server) { - if (!message.lrs_server.self) { - return false; - } - } - return true; - } - - /** - * Given a list of clusterNames (which may actually be the cluster name), - * for each watcher watching a name not on the list, call that watcher's - * onResourceDoesNotExist method. - * @param allClusterNames - */ - private handleMissingNames(allClusterNames: Set) { - for (const [clusterName, watcherList] of this.watchers.entries()) { - if (!allClusterNames.has(clusterName)) { - trace('Reporting CDS resource does not exist for clusterName ' + clusterName); - for (const watcher of watcherList) { - watcher.onResourceDoesNotExist(); - } - } - } - } - - handleResponses(responses: Cluster__Output[]): string | null { - for (const message of responses) { - if (!this.validateResponse(message)) { - trace('CDS validation failed for message ' + JSON.stringify(message)); - return 'CDS Error: Cluster validation failed'; - } - } - this.latestResponses = responses; - const allEdsServiceNames: Set = new Set(); - const allClusterNames: Set = new Set(); - for (const message of responses) { - allClusterNames.add(message.name); - const edsServiceName = message.eds_cluster_config?.service_name ?? ''; - allEdsServiceNames.add( - edsServiceName === '' ? message.name : edsServiceName - ); - const watchers = this.watchers.get(message.name) ?? []; - for (const watcher of watchers) { - watcher.onValidUpdate(message); - } - } - trace('Received CDS updates for cluster names ' + Array.from(allClusterNames)); - this.handleMissingNames(allClusterNames); - this.edsState.handleMissingNames(allEdsServiceNames); - return null; - } - - reportStreamError(status: StatusObject): void { - for (const watcherList of this.watchers.values()) { - for (const watcher of watcherList) { - watcher.onTransientError(status); - } - } - } -} - -class RdsState implements XdsStreamState { - versionInfo = ''; - nonce = ''; - - private routeConfigName: string | null = null; - - constructor( - private targetName: string, - private watcher: Watcher, - private updateResouceNames: () => void - ) {} - - getResourceNames(): string[] { - return this.routeConfigName ? [this.routeConfigName] : []; - } - - handleSingleMessage(message: RouteConfiguration__Output) { - for (const virtualHost of message.virtual_hosts) { - if (virtualHost.domains.indexOf(this.targetName) >= 0) { - const route = virtualHost.routes[virtualHost.routes.length - 1]; - if (route.match?.prefix === '' && route.route?.cluster) { - trace('Reporting RDS update for host ' + this.targetName + ' with cluster ' + route.route.cluster); - this.watcher.onValidUpdate({ - methodConfig: [], - loadBalancingConfig: [ - new CdsLoadBalancingConfig(route.route.cluster) - ], - }); - return; - } else { - trace('Discarded matching route with prefix ' + route.match?.prefix + ' and cluster ' + route.route?.cluster); - } - } - } - trace('Reporting RDS resource does not exist from domain lists ' + message.virtual_hosts.map(virtualHost => virtualHost.domains)); - /* If none of the routes match the one we are looking for, bubble up an - * error. */ - this.watcher.onResourceDoesNotExist(); - } - - handleResponses(responses: RouteConfiguration__Output[]): string | null { - trace('Received RDS response with route config names ' + responses.map(message => message.name)); - if (this.routeConfigName !== null) { - for (const message of responses) { - if (message.name === this.routeConfigName) { - this.handleSingleMessage(message); - return null; - } - } - } - return null; - } - - setRouteConfigName(name: string | null) { - const oldName = this.routeConfigName; - this.routeConfigName = name; - if (name !== oldName) { - this.updateResouceNames(); - } - } - - reportStreamError(status: StatusObject): void { - this.watcher.onTransientError(status); - } -} - -class LdsState implements XdsStreamState { - versionInfo = ''; - nonce = ''; - - constructor(private targetName: string, private rdsState: RdsState) {} - - getResourceNames(): string[] { - return [this.targetName]; - } - - private validateResponse(message: Listener__Output): boolean { - if ( - !( - message.api_listener?.api_listener && - protoLoader.isAnyExtension(message.api_listener.api_listener) && - message.api_listener?.api_listener['@type'] === - HTTP_CONNECTION_MANGER_TYPE_URL - ) - ) { - return false; - } - const httpConnectionManager = message.api_listener - ?.api_listener as protoLoader.AnyExtension & - HttpConnectionManager__Output; - switch (httpConnectionManager.route_specifier) { - case 'rds': - return !!httpConnectionManager.rds?.config_source?.ads; - case 'route_config': - return true; - } - return false; - } - - handleResponses(responses: Listener__Output[]): string | null { - trace('Received LDS update with names ' + responses.map(message => message.name)); - for (const message of responses) { - if (message.name === this.targetName) { - if (this.validateResponse(message)) { - // The validation step ensures that this is correct - const httpConnectionManager = message.api_listener! - .api_listener as protoLoader.AnyExtension & - HttpConnectionManager__Output; - switch (httpConnectionManager.route_specifier) { - case 'rds': - trace('Received LDS update with RDS route config name ' + httpConnectionManager.rds!.route_config_name); - this.rdsState.setRouteConfigName( - httpConnectionManager.rds!.route_config_name - ); - break; - case 'route_config': - trace('Received LDS update with route configuration'); - this.rdsState.setRouteConfigName(null); - this.rdsState.handleSingleMessage( - httpConnectionManager.route_config! - ); - break; - default: - // The validation rules should prevent this - } - } else { - trace('LRS validation error for message ' + JSON.stringify(message)); - return 'LRS Error: Listener validation failed'; - } - } - } - return null; - } - - reportStreamError(status: StatusObject): void { - // Nothing to do here - } -} - interface AdsState { [EDS_TYPE_URL]: EdsState; [CDS_TYPE_URL]: CdsState; @@ -728,21 +283,19 @@ export class XdsClient { private adsBackoff: BackoffTimeout; private lrsBackoff: BackoffTimeout; - constructor( - targetName: string, - serviceConfigWatcher: Watcher, - channelOptions: ChannelOptions - ) { + constructor() { const edsState = new EdsState(() => { this.updateNames(EDS_TYPE_URL); }); const cdsState = new CdsState(edsState, () => { this.updateNames(CDS_TYPE_URL); }); - const rdsState = new RdsState(targetName, serviceConfigWatcher, () => { + const rdsState = new RdsState(() => { this.updateNames(RDS_TYPE_URL); }); - const ldsState = new LdsState(targetName, rdsState); + const ldsState = new LdsState(rdsState, () => { + this.updateNames(LDS_TYPE_URL); + }); this.adsState = { [EDS_TYPE_URL]: edsState, [CDS_TYPE_URL]: cdsState, @@ -750,26 +303,10 @@ export class XdsClient { [LDS_TYPE_URL]: ldsState, }; - const channelArgs = { ...channelOptions }; - const channelArgsToRemove = [ - /* The SSL target name override corresponds to the target, and this - * client has its own target */ - 'grpc.ssl_target_name_override', - /* The default authority also corresponds to the target */ - 'grpc.default_authority', - /* This client will have its own specific keepalive time setting */ - 'grpc.keepalive_time_ms', - /* The service config specifies the load balancing policy. This channel - * needs its own separate load balancing policy setting. In particular, - * recursively using an xDS load balancer for the xDS client would be - * bad */ - 'grpc.service_config', - ]; - for (const arg of channelArgsToRemove) { - delete channelArgs[arg]; + const channelArgs = { + // 5 minutes + 'grpc.keepalive_time_ms': 5 * 60 * 1000 } - // 5 minutes - channelArgs['grpc.keepalive_time_ms'] = 5 * 60 * 1000; this.adsBackoff = new BackoffTimeout(() => { this.maybeStartAdsStream(); @@ -823,14 +360,12 @@ export class XdsClient { channelCreds, channelArgs ); - this.maybeStartAdsStream(); this.lrsClient = new protoDefinitions.envoy.service.load_stats.v2.LoadReportingService( bootstrapInfo.xdsServers[0].serverUri, channelCreds, {channelOverride: this.adsClient.getChannel()} ); - this.maybeStartLrsStream(); }, (error) => { trace('Failed to initialize xDS Client. ' + error.message); @@ -986,6 +521,16 @@ export class XdsClient { } private updateNames(typeUrl: AdsTypeUrl) { + if (this.adsState[EDS_TYPE_URL].getResourceNames().length === 0 && + this.adsState[CDS_TYPE_URL].getResourceNames().length === 0 && + this.adsState[RDS_TYPE_URL].getResourceNames().length === 0 && + this.adsState[LDS_TYPE_URL].getResourceNames().length === 0) { + this.adsCall?.end(); + this.lrsCall?.end(); + return; + } + this.maybeStartAdsStream(); + this.maybeStartLrsStream(); trace('Sending update for type URL ' + typeUrl + ' with names ' + this.adsState[typeUrl].getResourceNames()); this.adsCall?.write({ node: this.adsNode!, @@ -1159,6 +704,26 @@ export class XdsClient { this.adsState[CDS_TYPE_URL].removeWatcher(clusterName, watcher); } + addRouteWatcher(routeConfigName: string, watcher: Watcher) { + trace('Watcher added for route ' + routeConfigName); + this.adsState[RDS_TYPE_URL].addWatcher(routeConfigName, watcher); + } + + removeRouteWatcher(routeConfigName: string, watcher: Watcher) { + trace('Watcher removed for route ' + routeConfigName); + this.adsState[RDS_TYPE_URL].removeWatcher(routeConfigName, watcher); + } + + addListenerWatcher(targetName: string, watcher: Watcher) { + trace('Watcher added for listener ' + targetName); + this.adsState[LDS_TYPE_URL].addWatcher(targetName, watcher); + } + + removeListenerWatcher(targetName: string, watcher: Watcher) { + trace('Watcher removed for listener ' + targetName); + this.adsState[LDS_TYPE_URL].removeWatcher(targetName, watcher); + } + /** * * @param lrsServer The target name of the server to send stats to. An empty @@ -1241,7 +806,7 @@ export class XdsClient { }; } - shutdown(): void { + private shutdown(): void { this.adsCall?.cancel(); this.adsClient?.close(); this.lrsCall?.cancel(); @@ -1249,3 +814,12 @@ export class XdsClient { this.hasShutdown = true; } } + +let singletonXdsClient: XdsClient | null = null; + +export function getSingletonXdsClient(): XdsClient { + if (singletonXdsClient === null) { + singletonXdsClient = new XdsClient(); + } + return singletonXdsClient; +} \ No newline at end of file diff --git a/packages/grpc-js-xds/src/xds-stream-state/lds-state.ts b/packages/grpc-js-xds/src/xds-stream-state/lds-state.ts index c5db3bfa..55471272 100644 --- a/packages/grpc-js-xds/src/xds-stream-state/lds-state.ts +++ b/packages/grpc-js-xds/src/xds-stream-state/lds-state.ts @@ -19,7 +19,7 @@ import * as protoLoader from '@grpc/proto-loader'; import { experimental, logVerbosity, StatusObject } from "@grpc/grpc-js"; import { Listener__Output } from "../generated/envoy/api/v2/Listener"; import { RdsState } from "./rds-state"; -import { XdsStreamState } from "./xds-stream-state"; +import { Watcher, XdsStreamState } from "./xds-stream-state"; import { HttpConnectionManager__Output } from '../generated/envoy/config/filter/network/http_connection_manager/v2/HttpConnectionManager'; const TRACER_NAME = 'xds_client'; @@ -35,10 +35,60 @@ export class LdsState implements XdsStreamState { versionInfo = ''; nonce = ''; - constructor(private targetName: string, private rdsState: RdsState) {} + private watchers: Map[]> = new Map[]>(); + private latestResponses: Listener__Output[] = []; + + constructor(private rdsState: RdsState, private updateResourceNames: () => void) {} + + addWatcher(targetName: string, watcher: Watcher) { + trace('Adding RDS watcher for targetName ' + targetName); + let watchersEntry = this.watchers.get(targetName); + let addedServiceName = false; + if (watchersEntry === undefined) { + addedServiceName = true; + watchersEntry = []; + this.watchers.set(targetName, watchersEntry); + } + watchersEntry.push(watcher); + + /* If we have already received an update for the requested edsServiceName, + * immediately pass that update along to the watcher */ + for (const message of this.latestResponses) { + if (message.name === targetName) { + /* These updates normally occur asynchronously, so we ensure that + * the same happens here */ + process.nextTick(() => { + trace('Reporting existing RDS update for new watcher for targetName ' + targetName); + watcher.onValidUpdate(message); + }); + } + } + if (addedServiceName) { + this.updateResourceNames(); + } + } + + removeWatcher(targetName: string, watcher: Watcher): void { + trace('Removing RDS watcher for targetName ' + targetName); + const watchersEntry = this.watchers.get(targetName); + let removedServiceName = false; + if (watchersEntry !== undefined) { + const entryIndex = watchersEntry.indexOf(watcher); + if (entryIndex >= 0) { + watchersEntry.splice(entryIndex, 1); + } + if (watchersEntry.length === 0) { + removedServiceName = true; + this.watchers.delete(targetName); + } + } + if (removedServiceName) { + this.updateResourceNames(); + } + } getResourceNames(): string[] { - return [this.targetName]; + return Array.from(this.watchers.keys()); } private validateResponse(message: Listener__Output): boolean { @@ -59,47 +109,47 @@ export class LdsState implements XdsStreamState { case 'rds': return !!httpConnectionManager.rds?.config_source?.ads; case 'route_config': - return true; + return this.rdsState.validateResponse(httpConnectionManager.route_config!); } return false; } - handleResponses(responses: Listener__Output[]): string | null { - trace('Received LDS update with names ' + responses.map(message => message.name)); - for (const message of responses) { - if (message.name === this.targetName) { - if (this.validateResponse(message)) { - // The validation step ensures that this is correct - const httpConnectionManager = message.api_listener! - .api_listener as protoLoader.AnyExtension & - HttpConnectionManager__Output; - switch (httpConnectionManager.route_specifier) { - case 'rds': - trace('Received LDS update with RDS route config name ' + httpConnectionManager.rds!.route_config_name); - this.rdsState.setRouteConfigName( - httpConnectionManager.rds!.route_config_name - ); - break; - case 'route_config': - trace('Received LDS update with route configuration'); - this.rdsState.setRouteConfigName(null); - this.rdsState.handleSingleMessage( - httpConnectionManager.route_config! - ); - break; - default: - // The validation rules should prevent this - } - } else { - trace('LRS validation error for message ' + JSON.stringify(message)); - return 'LRS Error: Listener validation failed'; + private handleMissingNames(allTargetNames: Set) { + for (const [targetName, watcherList] of this.watchers.entries()) { + if (!allTargetNames.has(targetName)) { + for (const watcher of watcherList) { + watcher.onResourceDoesNotExist(); } } } + } + + handleResponses(responses: Listener__Output[]): string | null { + for (const message of responses) { + if (!this.validateResponse(message)) { + trace('LDS validation failed for message ' + JSON.stringify(message)); + return 'LDS Error: Route validation failed'; + } + } + this.latestResponses = responses; + const allTargetNames = new Set(); + for (const message of responses) { + allTargetNames.add(message.name); + const watchers = this.watchers.get(message.name) ?? []; + for (const watcher of watchers) { + watcher.onValidUpdate(message); + } + } + trace('Received RDS response with route config names ' + Array.from(allTargetNames)); + this.handleMissingNames(allTargetNames); return null; } reportStreamError(status: StatusObject): void { - // Nothing to do here + for (const watcherList of this.watchers.values()) { + for (const watcher of watcherList) { + watcher.onTransientError(status); + } + } } } \ No newline at end of file diff --git a/packages/grpc-js-xds/src/xds-stream-state/rds-state.ts b/packages/grpc-js-xds/src/xds-stream-state/rds-state.ts index 18268587..2ac924d9 100644 --- a/packages/grpc-js-xds/src/xds-stream-state/rds-state.ts +++ b/packages/grpc-js-xds/src/xds-stream-state/rds-state.ts @@ -16,6 +16,7 @@ */ import { experimental, logVerbosity, StatusObject } from "@grpc/grpc-js"; +import { GRPC_XDS_EXPERIMENTAL_ROUTING } from "../environment"; import { RouteConfiguration__Output } from "../generated/envoy/api/v2/RouteConfiguration"; import { CdsLoadBalancingConfig } from "../load-balancer-cds"; import { Watcher, XdsStreamState } from "./xds-stream-state"; @@ -27,68 +28,164 @@ function trace(text: string): void { experimental.trace(logVerbosity.DEBUG, TRACER_NAME, text); } +const SUPPORTED_PATH_SPECIFIERS = ['prefix', 'path', 'safe_regex']; +const SUPPPORTED_HEADER_MATCH_SPECIFIERS = [ + 'exact_match', + 'safe_regex_match', + 'range_match', + 'present_match', + 'prefix_match', + 'suffix_match']; +const SUPPORTED_CLUSTER_SPECIFIERS = ['cluster', 'weighted_clusters', 'cluster_header']; + export class RdsState implements XdsStreamState { versionInfo = ''; nonce = ''; - private routeConfigName: string | null = null; + private watchers: Map[]> = new Map[]>(); + private latestResponses: RouteConfiguration__Output[] = []; - constructor( - private targetName: string, - private watcher: Watcher, - private updateResouceNames: () => void - ) {} + constructor(private updateResourceNames: () => void) {} - getResourceNames(): string[] { - return this.routeConfigName ? [this.routeConfigName] : []; + addWatcher(routeConfigName: string, watcher: Watcher) { + trace('Adding RDS watcher for routeConfigName ' + routeConfigName); + let watchersEntry = this.watchers.get(routeConfigName); + let addedServiceName = false; + if (watchersEntry === undefined) { + addedServiceName = true; + watchersEntry = []; + this.watchers.set(routeConfigName, watchersEntry); + } + watchersEntry.push(watcher); + + /* If we have already received an update for the requested edsServiceName, + * immediately pass that update along to the watcher */ + for (const message of this.latestResponses) { + if (message.name === routeConfigName) { + /* These updates normally occur asynchronously, so we ensure that + * the same happens here */ + process.nextTick(() => { + trace('Reporting existing RDS update for new watcher for routeConfigName ' + routeConfigName); + watcher.onValidUpdate(message); + }); + } + } + if (addedServiceName) { + this.updateResourceNames(); + } } - handleSingleMessage(message: RouteConfiguration__Output) { - for (const virtualHost of message.virtual_hosts) { - if (virtualHost.domains.indexOf(this.targetName) >= 0) { - const route = virtualHost.routes[virtualHost.routes.length - 1]; - if (route.match?.prefix === '' && route.route?.cluster) { - trace('Reporting RDS update for host ' + this.targetName + ' with cluster ' + route.route.cluster); - this.watcher.onValidUpdate({ - methodConfig: [], - loadBalancingConfig: [ - new CdsLoadBalancingConfig(route.route.cluster) - ], - }); - return; - } else { - trace('Discarded matching route with prefix ' + route.match?.prefix + ' and cluster ' + route.route?.cluster); + removeWatcher(routeConfigName: string, watcher: Watcher): void { + trace('Removing RDS watcher for routeConfigName ' + routeConfigName); + const watchersEntry = this.watchers.get(routeConfigName); + let removedServiceName = false; + if (watchersEntry !== undefined) { + const entryIndex = watchersEntry.indexOf(watcher); + if (entryIndex >= 0) { + watchersEntry.splice(entryIndex, 1); + } + if (watchersEntry.length === 0) { + removedServiceName = true; + this.watchers.delete(routeConfigName); + } + } + if (removedServiceName) { + this.updateResourceNames(); + } + } + + getResourceNames(): string[] { + return Array.from(this.watchers.keys()); + } + + validateResponse(message: RouteConfiguration__Output): boolean { + if (GRPC_XDS_EXPERIMENTAL_ROUTING) { + // https://github.com/grpc/proposal/blob/master/A28-xds-traffic-splitting-and-routing.md#response-validation + for (const virtualHost of message.virtual_hosts) { + for (const domainPattern of virtualHost.domains) { + const starIndex = domainPattern.indexOf('*'); + const lastStarIndex = domainPattern.lastIndexOf('*'); + // A domain pattern can have at most one wildcard * + if (starIndex !== lastStarIndex) { + return false; + } + // A wildcard * can either be absent or at the beginning or end of the pattern + if (!(starIndex === -1 || starIndex === 0 || starIndex === domainPattern.length - 1)) { + return false; + } + } + for (const route of virtualHost.routes) { + const match = route.match; + if (!match) { + return false; + } + if (SUPPORTED_PATH_SPECIFIERS.indexOf(match.path_specifier) < 0) { + return false; + } + for (const headers of match.headers) { + if (SUPPPORTED_HEADER_MATCH_SPECIFIERS.indexOf(headers.header_match_specifier) < 0) { + return false; + } + } + if (route.action !== 'route') { + return false; + } + if ((route.route === undefined) || SUPPORTED_CLUSTER_SPECIFIERS.indexOf(route.route.cluster_specifier) < 0) { + return false; + } + if (route.route!.cluster_specifier === 'weighted_clusters') { + let weightSum = 0; + for (const clusterWeight of route.route.weighted_clusters!.clusters) { + weightSum += clusterWeight.weight?.value ?? 0; + } + if (weightSum !== route.route.weighted_clusters!.total_weight?.value ?? 100) { + return false; + } + } + } + } + return true; + } else { + return true; + } + } + + private handleMissingNames(allRouteConfigNames: Set) { + for (const [routeConfigName, watcherList] of this.watchers.entries()) { + if (!allRouteConfigNames.has(routeConfigName)) { + for (const watcher of watcherList) { + watcher.onResourceDoesNotExist(); } } } - trace('Reporting RDS resource does not exist from domain lists ' + message.virtual_hosts.map(virtualHost => virtualHost.domains)); - /* If none of the routes match the one we are looking for, bubble up an - * error. */ - this.watcher.onResourceDoesNotExist(); } handleResponses(responses: RouteConfiguration__Output[]): string | null { - trace('Received RDS response with route config names ' + responses.map(message => message.name)); - if (this.routeConfigName !== null) { - for (const message of responses) { - if (message.name === this.routeConfigName) { - this.handleSingleMessage(message); - return null; - } + for (const message of responses) { + if (!this.validateResponse(message)) { + trace('RDS validation failed for message ' + JSON.stringify(message)); + return 'RDS Error: Route validation failed'; } } + this.latestResponses = responses; + const allRouteConfigNames = new Set(); + for (const message of responses) { + allRouteConfigNames.add(message.name); + const watchers = this.watchers.get(message.name) ?? []; + for (const watcher of watchers) { + watcher.onValidUpdate(message); + } + } + trace('Received RDS response with route config names ' + Array.from(allRouteConfigNames)); + this.handleMissingNames(allRouteConfigNames); return null; } - setRouteConfigName(name: string | null) { - const oldName = this.routeConfigName; - this.routeConfigName = name; - if (name !== oldName) { - this.updateResouceNames(); + reportStreamError(status: StatusObject): void { + for (const watcherList of this.watchers.values()) { + for (const watcher of watcherList) { + watcher.onTransientError(status); + } } } - - reportStreamError(status: StatusObject): void { - this.watcher.onTransientError(status); - } } \ No newline at end of file diff --git a/packages/grpc-js/src/experimental.ts b/packages/grpc-js/src/experimental.ts index b88f124a..a0119ccb 100644 --- a/packages/grpc-js/src/experimental.ts +++ b/packages/grpc-js/src/experimental.ts @@ -1,5 +1,5 @@ export { trace } from './logging'; -export { Resolver, ResolverListener, registerResolver } from './resolver'; +export { Resolver, ResolverListener, registerResolver, ConfigSelector } from './resolver'; export { GrpcUri, uriToString } from './uri-parser'; export { ServiceConfig } from './service-config'; export { createGoogleDefaultCredentials } from './channel-credentials'; From 40242a4132ffd4297f24977cee5328b4dba49e35 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Mon, 1 Mar 2021 11:42:20 -0800 Subject: [PATCH 25/46] Use the new LB policy in the resolver --- packages/grpc-js-xds/src/environment.ts | 4 ++++ .../src/load-balancer-xds-cluster-manager.ts | 2 +- packages/grpc-js-xds/src/resolver-xds.ts | 15 ++++++++++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/grpc-js-xds/src/environment.ts b/packages/grpc-js-xds/src/environment.ts index 67fc531e..56d233dd 100644 --- a/packages/grpc-js-xds/src/environment.ts +++ b/packages/grpc-js-xds/src/environment.ts @@ -15,4 +15,8 @@ * */ +/** + * Environment variable protection for traffic splitting and routing + * https://github.com/grpc/proposal/blob/master/A28-xds-traffic-splitting-and-routing.md#xds-resolver-and-xds-client + */ export const GRPC_XDS_EXPERIMENTAL_ROUTING = (process.env.GRPC_XDS_EXPERIMENTAL_ROUTING === 'true'); \ No newline at end of file diff --git a/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts b/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts index f8d2de25..6faa0505 100644 --- a/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts +++ b/packages/grpc-js-xds/src/load-balancer-xds-cluster-manager.ts @@ -44,7 +44,7 @@ interface ClusterManagerChild { child_policy: LoadBalancingConfig[]; } -class XdsClusterManagerLoadBalancingConfig implements LoadBalancingConfig { +export class XdsClusterManagerLoadBalancingConfig implements LoadBalancingConfig { getLoadBalancerName(): string { return TYPE_NAME; } diff --git a/packages/grpc-js-xds/src/resolver-xds.ts b/packages/grpc-js-xds/src/resolver-xds.ts index b3c78ada..99bbd58c 100644 --- a/packages/grpc-js-xds/src/resolver-xds.ts +++ b/packages/grpc-js-xds/src/resolver-xds.ts @@ -37,6 +37,7 @@ import { RouteMatch__Output } from './generated/envoy/api/v2/route/RouteMatch'; import { HeaderMatcher__Output } from './generated/envoy/api/v2/route/HeaderMatcher'; import ConfigSelector = experimental.ConfigSelector; import LoadBalancingConfig = experimental.LoadBalancingConfig; +import { XdsClusterManagerLoadBalancingConfig } from './load-balancer-xds-cluster-manager'; const TRACER_NAME = 'xds_resolver'; @@ -376,7 +377,8 @@ class XdsResolver implements Resolver { const routeMatcher = getPredicateForMatcher(route.match!); matchList.push({matcher: routeMatcher, action: routeAction}); } - // Mark clusters that are not in this config, and remove ones with no references + /* Mark clusters that are not in this route config, and remove ones with + * no references */ for (const [name, refCount] of Array.from(this.clusterRefcounts.entries())) { if (!allConfigClusters.has(name)) { refCount.inLastConfig = false; @@ -385,6 +387,7 @@ class XdsResolver implements Resolver { } } } + // Add any new clusters from this route config for (const name of allConfigClusters) { if (this.clusterRefcounts.has(name)) { this.clusterRefcounts.get(name)!.inLastConfig = true; @@ -410,7 +413,7 @@ class XdsResolver implements Resolver { } return { methodConfig: {name: []}, - // pickInformation won't be used here, but it's set because of some TypeScript weirdness + // cluster won't be used here, but it's set because of some TypeScript weirdness pickInformation: {cluster: ''}, status: status.UNAVAILABLE }; @@ -419,8 +422,14 @@ class XdsResolver implements Resolver { for (const clusterName of this.clusterRefcounts.keys()) { clusterConfigMap.set(clusterName, {child_policy: [new CdsLoadBalancingConfig(clusterName)]}); } - // TODO: Create xdsClusterManagerLoadBalancingConfig and report successful resolution + const lbPolicyConfig = new XdsClusterManagerLoadBalancingConfig(clusterConfigMap); + const serviceConfig: ServiceConfig = { + methodConfig: [], + loadBalancingConfig: [lbPolicyConfig] + } + this.listener.onSuccessfulResolution([], serviceConfig, null, configSelector, {}); } else { + // !GRPC_XDS_EXPERIMENTAL_ROUTING for (const virtualHost of routeConfig.virtual_hosts) { if (virtualHost.domains.indexOf(this.target.path) >= 0) { const route = virtualHost.routes[virtualHost.routes.length - 1]; From 66d3f352639e0eb6d3f28866295ce2c5eeda3fc2 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Mon, 1 Mar 2021 11:54:20 -0800 Subject: [PATCH 26/46] Enable traffic splitting xds interop test --- packages/grpc-js-xds/scripts/xds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grpc-js-xds/scripts/xds.sh b/packages/grpc-js-xds/scripts/xds.sh index bbfc3056..a1bd7d40 100644 --- a/packages/grpc-js-xds/scripts/xds.sh +++ b/packages/grpc-js-xds/scripts/xds.sh @@ -52,7 +52,7 @@ GRPC_NODE_TRACE=xds_client,xds_resolver,cds_balancer,eds_balancer,priority,weigh GRPC_NODE_VERBOSITY=DEBUG \ NODE_XDS_INTEROP_VERBOSITY=1 \ python3 grpc/tools/run_tests/run_xds_tests.py \ - --test_case="backends_restart,change_backend_service,gentle_failover,ping_pong,remove_instance_group,round_robin,secondary_locality_gets_no_requests_on_partial_primary_failure,secondary_locality_gets_requests_on_primary_failure" \ + --test_case="all" \ --project_id=grpc-testing \ --source_image=projects/grpc-testing/global/images/xds-test-server-2 \ --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \ From 2b451fdfba1e53ae6d0c682e5bda06a8c2453d64 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Mon, 1 Mar 2021 11:55:15 -0800 Subject: [PATCH 27/46] Temporarily borrow linux test job for xds tests --- test/kokoro/linux.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/kokoro/linux.cfg b/test/kokoro/linux.cfg index f40e6db4..16a7dc7d 100644 --- a/test/kokoro/linux.cfg +++ b/test/kokoro/linux.cfg @@ -15,10 +15,10 @@ # Config file for Kokoro (in protobuf text format) # Location of the continuous shell script in repository. -build_file: "grpc-node/test/kokoro.sh" -timeout_mins: 60 +build_file: "grpc-node/packages/grpc-js-xds/scripts/xds.sh" +timeout_mins: 120 action { define_artifacts { - regex: "github/grpc-node/reports/**/sponge_log.xml" + regex: "github/grpc/reports/**" } } From f6505b50db78c29aa4854e6b5d454bda26c63c0c Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Mon, 1 Mar 2021 13:03:13 -0800 Subject: [PATCH 28/46] Enable routing feature in xDS tests --- packages/grpc-js-xds/scripts/xds.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/grpc-js-xds/scripts/xds.sh b/packages/grpc-js-xds/scripts/xds.sh index a1bd7d40..d76eaf4f 100644 --- a/packages/grpc-js-xds/scripts/xds.sh +++ b/packages/grpc-js-xds/scripts/xds.sh @@ -51,6 +51,7 @@ grpc/tools/run_tests/helper_scripts/prep_xds.sh GRPC_NODE_TRACE=xds_client,xds_resolver,cds_balancer,eds_balancer,priority,weighted_target,round_robin,resolving_load_balancer,subchannel,keepalive,dns_resolver \ GRPC_NODE_VERBOSITY=DEBUG \ NODE_XDS_INTEROP_VERBOSITY=1 \ + GRPC_XDS_EXPERIMENTAL_ROUTING=true \ python3 grpc/tools/run_tests/run_xds_tests.py \ --test_case="all" \ --project_id=grpc-testing \ From ca4b8f40c9c5562c83607b3ba0a184b26c1535a6 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Tue, 2 Mar 2021 12:51:23 -0800 Subject: [PATCH 29/46] Start ADS and LRS streams at client startup if watchers have been added --- packages/grpc-js-xds/src/xds-client.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/grpc-js-xds/src/xds-client.ts b/packages/grpc-js-xds/src/xds-client.ts index 83cd86cc..c928d5c0 100644 --- a/packages/grpc-js-xds/src/xds-client.ts +++ b/packages/grpc-js-xds/src/xds-client.ts @@ -360,12 +360,14 @@ export class XdsClient { channelCreds, channelArgs ); + this.maybeStartAdsStream(); this.lrsClient = new protoDefinitions.envoy.service.load_stats.v2.LoadReportingService( bootstrapInfo.xdsServers[0].serverUri, channelCreds, {channelOverride: this.adsClient.getChannel()} ); + this.maybeStartLrsStream(); }, (error) => { trace('Failed to initialize xDS Client. ' + error.message); @@ -439,6 +441,12 @@ export class XdsClient { if (this.hasShutdown) { return; } + if (this.adsState[EDS_TYPE_URL].getResourceNames().length === 0 && + this.adsState[CDS_TYPE_URL].getResourceNames().length === 0 && + this.adsState[RDS_TYPE_URL].getResourceNames().length === 0 && + this.adsState[LDS_TYPE_URL].getResourceNames().length === 0) { + return; + } trace('Starting ADS stream'); // Backoff relative to when we start the request this.adsBackoff.runOnce(); @@ -558,6 +566,12 @@ export class XdsClient { if (this.hasShutdown) { return; } + if (this.adsState[EDS_TYPE_URL].getResourceNames().length === 0 && + this.adsState[CDS_TYPE_URL].getResourceNames().length === 0 && + this.adsState[RDS_TYPE_URL].getResourceNames().length === 0 && + this.adsState[LDS_TYPE_URL].getResourceNames().length === 0) { + return; + } trace('Starting LRS stream'); From a72626558049eacd72d95f8139dabeb47edf494c Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Wed, 3 Mar 2021 13:15:57 -0800 Subject: [PATCH 30/46] Remove checks for now-unused xdsClient attribute --- packages/grpc-js-xds/src/load-balancer-cds.ts | 4 ---- packages/grpc-js-xds/src/load-balancer-eds.ts | 4 ---- packages/grpc-js-xds/src/load-balancer-lrs.ts | 7 ++----- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/grpc-js-xds/src/load-balancer-cds.ts b/packages/grpc-js-xds/src/load-balancer-cds.ts index 9a372588..d0fe2338 100644 --- a/packages/grpc-js-xds/src/load-balancer-cds.ts +++ b/packages/grpc-js-xds/src/load-balancer-cds.ts @@ -121,10 +121,6 @@ export class CdsLoadBalancer implements LoadBalancer { trace('Discarding address list update with unrecognized config ' + JSON.stringify(lbConfig, undefined, 2)); return; } - if (!(attributes.xdsClient instanceof XdsClient)) { - trace('Discarding address list update missing xdsClient attribute'); - return; - } trace('Received update with config ' + JSON.stringify(lbConfig, undefined, 2)); this.latestAttributes = attributes; diff --git a/packages/grpc-js-xds/src/load-balancer-eds.ts b/packages/grpc-js-xds/src/load-balancer-eds.ts index 35da2a46..efc67ffe 100644 --- a/packages/grpc-js-xds/src/load-balancer-eds.ts +++ b/packages/grpc-js-xds/src/load-balancer-eds.ts @@ -427,10 +427,6 @@ export class EdsLoadBalancer implements LoadBalancer { trace('Discarding address list update with unrecognized config ' + JSON.stringify(lbConfig.toJsonObject(), undefined, 2)); return; } - if (!(attributes.xdsClient instanceof XdsClient)) { - trace('Discarding address list update missing xdsClient attribute'); - return; - } trace('Received update with config: ' + JSON.stringify(lbConfig, undefined, 2)); this.lastestConfig = lbConfig; this.latestAttributes = attributes; diff --git a/packages/grpc-js-xds/src/load-balancer-lrs.ts b/packages/grpc-js-xds/src/load-balancer-lrs.ts index b6fa6809..a6bbcb87 100644 --- a/packages/grpc-js-xds/src/load-balancer-lrs.ts +++ b/packages/grpc-js-xds/src/load-balancer-lrs.ts @@ -18,7 +18,7 @@ import { connectivityState as ConnectivityState, StatusObject, status as Status, experimental } from '@grpc/grpc-js'; import { type } from 'os'; import { Locality__Output } from './generated/envoy/api/v2/core/Locality'; -import { XdsClusterLocalityStats, XdsClient } from './xds-client'; +import { XdsClusterLocalityStats, XdsClient, getSingletonXdsClient } from './xds-client'; import LoadBalancer = experimental.LoadBalancer; import ChannelControlHelper = experimental.ChannelControlHelper; import registerLoadBalancerType = experimental.registerLoadBalancerType; @@ -208,10 +208,7 @@ export class LrsLoadBalancer implements LoadBalancer { if (!(lbConfig instanceof LrsLoadBalancingConfig)) { return; } - if (!(attributes.xdsClient instanceof XdsClient)) { - return; - } - this.localityStatsReporter = attributes.xdsClient.addClusterLocalityStats( + this.localityStatsReporter = getSingletonXdsClient().addClusterLocalityStats( lbConfig.getLrsLoadReportingServerName(), lbConfig.getClusterName(), lbConfig.getEdsServiceName(), From fb8916cc6d202ebd39f78e85423d32a22e47e38e Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Thu, 4 Mar 2021 10:16:17 -0800 Subject: [PATCH 31/46] Enable path_matching and header_matching tests --- packages/grpc-js-xds/scripts/xds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grpc-js-xds/scripts/xds.sh b/packages/grpc-js-xds/scripts/xds.sh index d76eaf4f..30801797 100644 --- a/packages/grpc-js-xds/scripts/xds.sh +++ b/packages/grpc-js-xds/scripts/xds.sh @@ -53,7 +53,7 @@ GRPC_NODE_TRACE=xds_client,xds_resolver,cds_balancer,eds_balancer,priority,weigh NODE_XDS_INTEROP_VERBOSITY=1 \ GRPC_XDS_EXPERIMENTAL_ROUTING=true \ python3 grpc/tools/run_tests/run_xds_tests.py \ - --test_case="all" \ + --test_case="all,path_matching,header_matching" \ --project_id=grpc-testing \ --source_image=projects/grpc-testing/global/images/xds-test-server-2 \ --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \ From 08a359744d94319e492c900f80be063ec60b019c Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Thu, 4 Mar 2021 11:50:30 -0800 Subject: [PATCH 32/46] Add more detailed LRS tracing --- packages/grpc-js-xds/src/xds-client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grpc-js-xds/src/xds-client.ts b/packages/grpc-js-xds/src/xds-client.ts index c928d5c0..c42ce663 100644 --- a/packages/grpc-js-xds/src/xds-client.ts +++ b/packages/grpc-js-xds/src/xds-client.ts @@ -625,7 +625,6 @@ export class XdsClient { if (!this.lrsCall) { return; } - trace('Sending LRS stats'); const clusterStats: ClusterStats[] = []; for (const [ { clusterName, edsServiceName }, @@ -686,6 +685,7 @@ export class XdsClient { } } } + trace('Sending LRS stats ' + JSON.stringify(clusterStats, undefined, 2)); this.lrsCall.write({ node: this.lrsNode!, cluster_stats: clusterStats, From 76f4e3fef445eed2a100d68c1ebc73bd287dc627 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Thu, 4 Mar 2021 13:43:17 -0800 Subject: [PATCH 33/46] Disable path_matching and header_matching tests for now --- packages/grpc-js-xds/scripts/xds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grpc-js-xds/scripts/xds.sh b/packages/grpc-js-xds/scripts/xds.sh index 30801797..d76eaf4f 100644 --- a/packages/grpc-js-xds/scripts/xds.sh +++ b/packages/grpc-js-xds/scripts/xds.sh @@ -53,7 +53,7 @@ GRPC_NODE_TRACE=xds_client,xds_resolver,cds_balancer,eds_balancer,priority,weigh NODE_XDS_INTEROP_VERBOSITY=1 \ GRPC_XDS_EXPERIMENTAL_ROUTING=true \ python3 grpc/tools/run_tests/run_xds_tests.py \ - --test_case="all,path_matching,header_matching" \ + --test_case="all" \ --project_id=grpc-testing \ --source_image=projects/grpc-testing/global/images/xds-test-server-2 \ --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \ From 5ef5246375b2d45a495c6c3f4c5c03fbc0a02144 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Thu, 4 Mar 2021 13:53:02 -0800 Subject: [PATCH 34/46] Fix handling of LRS server name in EDS child config generation Also add more LRS logging --- packages/grpc-js-xds/src/load-balancer-eds.ts | 2 +- packages/grpc-js-xds/src/load-balancer-lrs.ts | 1 - packages/grpc-js-xds/src/xds-client.ts | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/grpc-js-xds/src/load-balancer-eds.ts b/packages/grpc-js-xds/src/load-balancer-eds.ts index efc67ffe..01e40f5a 100644 --- a/packages/grpc-js-xds/src/load-balancer-eds.ts +++ b/packages/grpc-js-xds/src/load-balancer-eds.ts @@ -377,7 +377,7 @@ export class EdsLoadBalancer implements LoadBalancer { validateLoadBalancingConfig({ round_robin: {} }), ]; let childPolicy: LoadBalancingConfig[]; - if (this.lastestConfig.getLrsLoadReportingServerName()) { + if (this.lastestConfig.getLrsLoadReportingServerName() !== undefined) { childPolicy = [new LrsLoadBalancingConfig(this.lastestConfig.getCluster(), this.lastestConfig.getEdsServiceName() ?? '', this.lastestConfig.getLrsLoadReportingServerName()!, localityObj.locality, endpointPickingPolicy)]; } else { childPolicy = endpointPickingPolicy; diff --git a/packages/grpc-js-xds/src/load-balancer-lrs.ts b/packages/grpc-js-xds/src/load-balancer-lrs.ts index a6bbcb87..0792b11c 100644 --- a/packages/grpc-js-xds/src/load-balancer-lrs.ts +++ b/packages/grpc-js-xds/src/load-balancer-lrs.ts @@ -16,7 +16,6 @@ */ import { connectivityState as ConnectivityState, StatusObject, status as Status, experimental } from '@grpc/grpc-js'; -import { type } from 'os'; import { Locality__Output } from './generated/envoy/api/v2/core/Locality'; import { XdsClusterLocalityStats, XdsClient, getSingletonXdsClient } from './xds-client'; import LoadBalancer = experimental.LoadBalancer; diff --git a/packages/grpc-js-xds/src/xds-client.ts b/packages/grpc-js-xds/src/xds-client.ts index c42ce663..8ecd5bb7 100644 --- a/packages/grpc-js-xds/src/xds-client.ts +++ b/packages/grpc-js-xds/src/xds-client.ts @@ -751,6 +751,7 @@ export class XdsClient { clusterName: string, edsServiceName: string ): XdsClusterDropStats { + trace('addClusterDropStats(lrsServer=' + lrsServer + ', clusterName=' + clusterName + ', edsServiceName=' + edsServiceName + ')'); if (lrsServer !== '') { return { addCallDropped: (category) => {}, @@ -774,6 +775,7 @@ export class XdsClient { edsServiceName: string, locality: Locality__Output ): XdsClusterLocalityStats { + trace('addClusterLocalityStats(lrsServer=' + lrsServer + ', clusterName=' + clusterName + ', edsServiceName=' + edsServiceName + ', locality=' + JSON.stringify(locality) + ')'); if (lrsServer !== '') { return { addCallStarted: () => {}, From dd22f8f499fc3831ca0bfd31e6b9aef4a00501fd Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Thu, 4 Mar 2021 18:35:12 -0800 Subject: [PATCH 35/46] Don't send status through the filter stack twice when receiving trailers --- packages/grpc-js/src/call-stream.ts | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/packages/grpc-js/src/call-stream.ts b/packages/grpc-js/src/call-stream.ts index 9c27aeb1..7d0abb94 100644 --- a/packages/grpc-js/src/call-stream.ts +++ b/packages/grpc-js/src/call-stream.ts @@ -415,21 +415,8 @@ export class Http2CallStream implements Call { ); } const status: StatusObject = { code, details, metadata }; - let finalStatus; - try { - // Attempt to assign final status. - finalStatus = this.filterStack.receiveTrailers(status); - } catch (error) { - // This is a no-op if the call was already ended when handling headers. - this.endCall({ - code: Status.INTERNAL, - details: 'Failed to process received status', - metadata: new Metadata(), - }); - return; - } // This is a no-op if the call was already ended when handling headers. - this.endCall(finalStatus); + this.endCall(status); } attachHttp2Stream( From e7eaeeb090f0c4c503ef7005e7014059880bdefa Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Fri, 5 Mar 2021 11:09:21 -0800 Subject: [PATCH 36/46] Revert "Temporarily borrow linux test job for xds tests" This reverts commit 2b451fdfba1e53ae6d0c682e5bda06a8c2453d64. --- test/kokoro/linux.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/kokoro/linux.cfg b/test/kokoro/linux.cfg index 16a7dc7d..f40e6db4 100644 --- a/test/kokoro/linux.cfg +++ b/test/kokoro/linux.cfg @@ -15,10 +15,10 @@ # Config file for Kokoro (in protobuf text format) # Location of the continuous shell script in repository. -build_file: "grpc-node/packages/grpc-js-xds/scripts/xds.sh" -timeout_mins: 120 +build_file: "grpc-node/test/kokoro.sh" +timeout_mins: 60 action { define_artifacts { - regex: "github/grpc/reports/**" + regex: "github/grpc-node/reports/**/sponge_log.xml" } } From c2d7e4addaffc8b171341a62bc7e0bbcbc0b2c18 Mon Sep 17 00:00:00 2001 From: sovlookup Date: Fri, 12 Mar 2021 19:39:06 +0800 Subject: [PATCH 37/46] load protobuf.js JSON descriptor --- packages/proto-loader/src/index.ts | 17 +++++++++++++++++ .../proto-loader/test/descriptor_type_test.ts | 9 +++++++++ .../proto-loader/test_protos/rpc.proto.json | 16 ++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 packages/proto-loader/test_protos/rpc.proto.json diff --git a/packages/proto-loader/src/index.ts b/packages/proto-loader/src/index.ts index c51bc011..237fd840 100644 --- a/packages/proto-loader/src/index.ts +++ b/packages/proto-loader/src/index.ts @@ -395,6 +395,23 @@ export function loadSync( return createPackageDefinition(root, options!); } +export function fromJSON( + json: Protobuf.INamespace, + root?: Protobuf.Root +): PackageDefinition { + const options: Options = json.options || {}; + const newRoot: Protobuf.Root = root || new Protobuf.Root(); + if (!!options.includeDirs) { + if (!Array.isArray(options.includeDirs)) { + throw new Error('The includeDirs option must be an array'); + } + addIncludePathResolver(newRoot, options.includeDirs as string[]); + } + const loadedRoot = Protobuf.Root.fromJSON(json, newRoot); + loadedRoot.resolveAll(); + return createPackageDefinition(newRoot, options!); +} + export function loadFileDescriptorSetFromBuffer( descriptorSet: Buffer, options?: Options diff --git a/packages/proto-loader/test/descriptor_type_test.ts b/packages/proto-loader/test/descriptor_type_test.ts index 32aca9fc..180681c1 100644 --- a/packages/proto-loader/test/descriptor_type_test.ts +++ b/packages/proto-loader/test/descriptor_type_test.ts @@ -102,6 +102,15 @@ describe('Descriptor types', () => { proto_loader.loadSync(`${TEST_PROTO_DIR}/well_known.proto`); }); + it('Can load JSON descriptors', () => { + // This is protobuf.js JSON descriptor + // https://github.com/protobufjs/protobuf.js#using-json-descriptors + const buffer = readFileSync(`${TEST_PROTO_DIR}/rpc.proto.json`); + const json = JSON.parse(buffer.toString()); + // This will throw if the rpc descriptor JSON cannot be decoded + proto_loader.fromJSON(json); + }); + it('Can load binary-encoded proto file descriptor sets', () => { const buffer = readFileSync(`${TEST_PROTO_DIR}/rpc.desc.bin`); // This will throw if the rpc descriptor cannot be decoded diff --git a/packages/proto-loader/test_protos/rpc.proto.json b/packages/proto-loader/test_protos/rpc.proto.json new file mode 100644 index 00000000..e49dec1d --- /dev/null +++ b/packages/proto-loader/test_protos/rpc.proto.json @@ -0,0 +1,16 @@ +{ + "nested": { + "awesomepackage": { + "nested": { + "AwesomeMessage": { + "fields": { + "awesomeField": { + "type": "string", + "id": 1 + } + } + } + } + } + } +} \ No newline at end of file From 2b0ebcfc6a136e761061cfe357960104888f2402 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Fri, 12 Mar 2021 14:59:01 -0800 Subject: [PATCH 38/46] grpc-js-xds: Add functionality to the xDS interop client --- .../grpc/testing/ClientConfigureRequest.ts | 68 +++++++++ .../grpc/testing/ClientConfigureResponse.ts | 14 ++ .../LoadBalancerAccumulatedStatsRequest.ts | 14 ++ .../LoadBalancerAccumulatedStatsResponse.ts | 78 ++++++++++ .../grpc/testing/LoadBalancerStatsService.ts | 22 +++ .../XdsUpdateClientConfigureService.ts | 37 +++++ .../grpc-js-xds/interop/generated/test.ts | 9 ++ .../grpc-js-xds/interop/xds-interop-client.ts | 138 +++++++++++++----- .../proto/grpc/testing/messages.proto | 56 +++++++ .../grpc-js-xds/proto/grpc/testing/test.proto | 10 ++ 10 files changed, 412 insertions(+), 34 deletions(-) create mode 100644 packages/grpc-js-xds/interop/generated/grpc/testing/ClientConfigureRequest.ts create mode 100644 packages/grpc-js-xds/interop/generated/grpc/testing/ClientConfigureResponse.ts create mode 100644 packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerAccumulatedStatsRequest.ts create mode 100644 packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerAccumulatedStatsResponse.ts create mode 100644 packages/grpc-js-xds/interop/generated/grpc/testing/XdsUpdateClientConfigureService.ts diff --git a/packages/grpc-js-xds/interop/generated/grpc/testing/ClientConfigureRequest.ts b/packages/grpc-js-xds/interop/generated/grpc/testing/ClientConfigureRequest.ts new file mode 100644 index 00000000..7f07e896 --- /dev/null +++ b/packages/grpc-js-xds/interop/generated/grpc/testing/ClientConfigureRequest.ts @@ -0,0 +1,68 @@ +// Original file: proto/grpc/testing/messages.proto + + +/** + * Metadata to be attached for the given type of RPCs. + */ +export interface _grpc_testing_ClientConfigureRequest_Metadata { + 'type'?: (_grpc_testing_ClientConfigureRequest_RpcType | keyof typeof _grpc_testing_ClientConfigureRequest_RpcType); + 'key'?: (string); + 'value'?: (string); +} + +/** + * Metadata to be attached for the given type of RPCs. + */ +export interface _grpc_testing_ClientConfigureRequest_Metadata__Output { + 'type': (keyof typeof _grpc_testing_ClientConfigureRequest_RpcType); + 'key': (string); + 'value': (string); +} + +// Original file: proto/grpc/testing/messages.proto + +/** + * Type of RPCs to send. + */ +export enum _grpc_testing_ClientConfigureRequest_RpcType { + EMPTY_CALL = 0, + UNARY_CALL = 1, +} + +/** + * Configurations for a test client. + */ +export interface ClientConfigureRequest { + /** + * The types of RPCs the client sends. + */ + 'types'?: (_grpc_testing_ClientConfigureRequest_RpcType | keyof typeof _grpc_testing_ClientConfigureRequest_RpcType)[]; + /** + * The collection of custom metadata to be attached to RPCs sent by the client. + */ + 'metadata'?: (_grpc_testing_ClientConfigureRequest_Metadata)[]; + /** + * The deadline to use, in seconds, for all RPCs. If unset or zero, the + * client will use the default from the command-line. + */ + 'timeout_sec'?: (number); +} + +/** + * Configurations for a test client. + */ +export interface ClientConfigureRequest__Output { + /** + * The types of RPCs the client sends. + */ + 'types': (keyof typeof _grpc_testing_ClientConfigureRequest_RpcType)[]; + /** + * The collection of custom metadata to be attached to RPCs sent by the client. + */ + 'metadata': (_grpc_testing_ClientConfigureRequest_Metadata__Output)[]; + /** + * The deadline to use, in seconds, for all RPCs. If unset or zero, the + * client will use the default from the command-line. + */ + 'timeout_sec': (number); +} diff --git a/packages/grpc-js-xds/interop/generated/grpc/testing/ClientConfigureResponse.ts b/packages/grpc-js-xds/interop/generated/grpc/testing/ClientConfigureResponse.ts new file mode 100644 index 00000000..df663240 --- /dev/null +++ b/packages/grpc-js-xds/interop/generated/grpc/testing/ClientConfigureResponse.ts @@ -0,0 +1,14 @@ +// Original file: proto/grpc/testing/messages.proto + + +/** + * Response for updating a test client's configuration. + */ +export interface ClientConfigureResponse { +} + +/** + * Response for updating a test client's configuration. + */ +export interface ClientConfigureResponse__Output { +} diff --git a/packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerAccumulatedStatsRequest.ts b/packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerAccumulatedStatsRequest.ts new file mode 100644 index 00000000..1aa9773c --- /dev/null +++ b/packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerAccumulatedStatsRequest.ts @@ -0,0 +1,14 @@ +// Original file: proto/grpc/testing/messages.proto + + +/** + * Request for retrieving a test client's accumulated stats. + */ +export interface LoadBalancerAccumulatedStatsRequest { +} + +/** + * Request for retrieving a test client's accumulated stats. + */ +export interface LoadBalancerAccumulatedStatsRequest__Output { +} diff --git a/packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerAccumulatedStatsResponse.ts b/packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerAccumulatedStatsResponse.ts new file mode 100644 index 00000000..91157ac4 --- /dev/null +++ b/packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerAccumulatedStatsResponse.ts @@ -0,0 +1,78 @@ +// Original file: proto/grpc/testing/messages.proto + + +export interface _grpc_testing_LoadBalancerAccumulatedStatsResponse_MethodStats { + /** + * The number of RPCs that were started for this method. + */ + 'rpcs_started'?: (number); + /** + * The number of RPCs that completed with each status for this method. The + * key is the integral value of a google.rpc.Code; the value is the count. + */ + 'result'?: ({[key: number]: number}); +} + +export interface _grpc_testing_LoadBalancerAccumulatedStatsResponse_MethodStats__Output { + /** + * The number of RPCs that were started for this method. + */ + 'rpcs_started': (number); + /** + * The number of RPCs that completed with each status for this method. The + * key is the integral value of a google.rpc.Code; the value is the count. + */ + 'result': ({[key: number]: number}); +} + +/** + * Accumulated stats for RPCs sent by a test client. + */ +export interface LoadBalancerAccumulatedStatsResponse { + /** + * The total number of RPCs have ever issued for each type. + * Deprecated: use stats_per_method.rpcs_started instead. + */ + 'num_rpcs_started_by_method'?: ({[key: string]: number}); + /** + * The total number of RPCs have ever completed successfully for each type. + * Deprecated: use stats_per_method.result instead. + */ + 'num_rpcs_succeeded_by_method'?: ({[key: string]: number}); + /** + * The total number of RPCs have ever failed for each type. + * Deprecated: use stats_per_method.result instead. + */ + 'num_rpcs_failed_by_method'?: ({[key: string]: number}); + /** + * Per-method RPC statistics. The key is the RpcType in string form; e.g. + * 'EMPTY_CALL' or 'UNARY_CALL' + */ + 'stats_per_method'?: ({[key: string]: _grpc_testing_LoadBalancerAccumulatedStatsResponse_MethodStats}); +} + +/** + * Accumulated stats for RPCs sent by a test client. + */ +export interface LoadBalancerAccumulatedStatsResponse__Output { + /** + * The total number of RPCs have ever issued for each type. + * Deprecated: use stats_per_method.rpcs_started instead. + */ + 'num_rpcs_started_by_method': ({[key: string]: number}); + /** + * The total number of RPCs have ever completed successfully for each type. + * Deprecated: use stats_per_method.result instead. + */ + 'num_rpcs_succeeded_by_method': ({[key: string]: number}); + /** + * The total number of RPCs have ever failed for each type. + * Deprecated: use stats_per_method.result instead. + */ + 'num_rpcs_failed_by_method': ({[key: string]: number}); + /** + * Per-method RPC statistics. The key is the RpcType in string form; e.g. + * 'EMPTY_CALL' or 'UNARY_CALL' + */ + 'stats_per_method'?: ({[key: string]: _grpc_testing_LoadBalancerAccumulatedStatsResponse_MethodStats__Output}); +} diff --git a/packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerStatsService.ts b/packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerStatsService.ts index aa4f409f..2c972726 100644 --- a/packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerStatsService.ts +++ b/packages/grpc-js-xds/interop/generated/grpc/testing/LoadBalancerStatsService.ts @@ -1,6 +1,8 @@ // Original file: proto/grpc/testing/test.proto import * as grpc from '@grpc/grpc-js' +import { LoadBalancerAccumulatedStatsRequest as _grpc_testing_LoadBalancerAccumulatedStatsRequest, LoadBalancerAccumulatedStatsRequest__Output as _grpc_testing_LoadBalancerAccumulatedStatsRequest__Output } from '../../grpc/testing/LoadBalancerAccumulatedStatsRequest'; +import { LoadBalancerAccumulatedStatsResponse as _grpc_testing_LoadBalancerAccumulatedStatsResponse, LoadBalancerAccumulatedStatsResponse__Output as _grpc_testing_LoadBalancerAccumulatedStatsResponse__Output } from '../../grpc/testing/LoadBalancerAccumulatedStatsResponse'; import { LoadBalancerStatsRequest as _grpc_testing_LoadBalancerStatsRequest, LoadBalancerStatsRequest__Output as _grpc_testing_LoadBalancerStatsRequest__Output } from '../../grpc/testing/LoadBalancerStatsRequest'; import { LoadBalancerStatsResponse as _grpc_testing_LoadBalancerStatsResponse, LoadBalancerStatsResponse__Output as _grpc_testing_LoadBalancerStatsResponse__Output } from '../../grpc/testing/LoadBalancerStatsResponse'; @@ -8,6 +10,21 @@ import { LoadBalancerStatsResponse as _grpc_testing_LoadBalancerStatsResponse, L * A service used to obtain stats for verifying LB behavior. */ export interface LoadBalancerStatsServiceClient extends grpc.Client { + /** + * Gets the accumulated stats for RPCs sent by a test client. + */ + GetClientAccumulatedStats(argument: _grpc_testing_LoadBalancerAccumulatedStatsRequest, metadata: grpc.Metadata, options: grpc.CallOptions, callback: (error?: grpc.ServiceError, result?: _grpc_testing_LoadBalancerAccumulatedStatsResponse__Output) => void): grpc.ClientUnaryCall; + GetClientAccumulatedStats(argument: _grpc_testing_LoadBalancerAccumulatedStatsRequest, metadata: grpc.Metadata, callback: (error?: grpc.ServiceError, result?: _grpc_testing_LoadBalancerAccumulatedStatsResponse__Output) => void): grpc.ClientUnaryCall; + GetClientAccumulatedStats(argument: _grpc_testing_LoadBalancerAccumulatedStatsRequest, options: grpc.CallOptions, callback: (error?: grpc.ServiceError, result?: _grpc_testing_LoadBalancerAccumulatedStatsResponse__Output) => void): grpc.ClientUnaryCall; + GetClientAccumulatedStats(argument: _grpc_testing_LoadBalancerAccumulatedStatsRequest, callback: (error?: grpc.ServiceError, result?: _grpc_testing_LoadBalancerAccumulatedStatsResponse__Output) => void): grpc.ClientUnaryCall; + /** + * Gets the accumulated stats for RPCs sent by a test client. + */ + getClientAccumulatedStats(argument: _grpc_testing_LoadBalancerAccumulatedStatsRequest, metadata: grpc.Metadata, options: grpc.CallOptions, callback: (error?: grpc.ServiceError, result?: _grpc_testing_LoadBalancerAccumulatedStatsResponse__Output) => void): grpc.ClientUnaryCall; + getClientAccumulatedStats(argument: _grpc_testing_LoadBalancerAccumulatedStatsRequest, metadata: grpc.Metadata, callback: (error?: grpc.ServiceError, result?: _grpc_testing_LoadBalancerAccumulatedStatsResponse__Output) => void): grpc.ClientUnaryCall; + getClientAccumulatedStats(argument: _grpc_testing_LoadBalancerAccumulatedStatsRequest, options: grpc.CallOptions, callback: (error?: grpc.ServiceError, result?: _grpc_testing_LoadBalancerAccumulatedStatsResponse__Output) => void): grpc.ClientUnaryCall; + getClientAccumulatedStats(argument: _grpc_testing_LoadBalancerAccumulatedStatsRequest, callback: (error?: grpc.ServiceError, result?: _grpc_testing_LoadBalancerAccumulatedStatsResponse__Output) => void): grpc.ClientUnaryCall; + /** * Gets the backend distribution for RPCs sent by a test client. */ @@ -29,6 +46,11 @@ export interface LoadBalancerStatsServiceClient extends grpc.Client { * A service used to obtain stats for verifying LB behavior. */ export interface LoadBalancerStatsServiceHandlers extends grpc.UntypedServiceImplementation { + /** + * Gets the accumulated stats for RPCs sent by a test client. + */ + GetClientAccumulatedStats(call: grpc.ServerUnaryCall<_grpc_testing_LoadBalancerAccumulatedStatsRequest__Output, _grpc_testing_LoadBalancerAccumulatedStatsResponse>, callback: grpc.sendUnaryData<_grpc_testing_LoadBalancerAccumulatedStatsResponse>): void; + /** * Gets the backend distribution for RPCs sent by a test client. */ diff --git a/packages/grpc-js-xds/interop/generated/grpc/testing/XdsUpdateClientConfigureService.ts b/packages/grpc-js-xds/interop/generated/grpc/testing/XdsUpdateClientConfigureService.ts new file mode 100644 index 00000000..693c37ec --- /dev/null +++ b/packages/grpc-js-xds/interop/generated/grpc/testing/XdsUpdateClientConfigureService.ts @@ -0,0 +1,37 @@ +// Original file: proto/grpc/testing/test.proto + +import * as grpc from '@grpc/grpc-js' +import { ClientConfigureRequest as _grpc_testing_ClientConfigureRequest, ClientConfigureRequest__Output as _grpc_testing_ClientConfigureRequest__Output } from '../../grpc/testing/ClientConfigureRequest'; +import { ClientConfigureResponse as _grpc_testing_ClientConfigureResponse, ClientConfigureResponse__Output as _grpc_testing_ClientConfigureResponse__Output } from '../../grpc/testing/ClientConfigureResponse'; + +/** + * A service to dynamically update the configuration of an xDS test client. + */ +export interface XdsUpdateClientConfigureServiceClient extends grpc.Client { + /** + * Update the tes client's configuration. + */ + Configure(argument: _grpc_testing_ClientConfigureRequest, metadata: grpc.Metadata, options: grpc.CallOptions, callback: (error?: grpc.ServiceError, result?: _grpc_testing_ClientConfigureResponse__Output) => void): grpc.ClientUnaryCall; + Configure(argument: _grpc_testing_ClientConfigureRequest, metadata: grpc.Metadata, callback: (error?: grpc.ServiceError, result?: _grpc_testing_ClientConfigureResponse__Output) => void): grpc.ClientUnaryCall; + Configure(argument: _grpc_testing_ClientConfigureRequest, options: grpc.CallOptions, callback: (error?: grpc.ServiceError, result?: _grpc_testing_ClientConfigureResponse__Output) => void): grpc.ClientUnaryCall; + Configure(argument: _grpc_testing_ClientConfigureRequest, callback: (error?: grpc.ServiceError, result?: _grpc_testing_ClientConfigureResponse__Output) => void): grpc.ClientUnaryCall; + /** + * Update the tes client's configuration. + */ + configure(argument: _grpc_testing_ClientConfigureRequest, metadata: grpc.Metadata, options: grpc.CallOptions, callback: (error?: grpc.ServiceError, result?: _grpc_testing_ClientConfigureResponse__Output) => void): grpc.ClientUnaryCall; + configure(argument: _grpc_testing_ClientConfigureRequest, metadata: grpc.Metadata, callback: (error?: grpc.ServiceError, result?: _grpc_testing_ClientConfigureResponse__Output) => void): grpc.ClientUnaryCall; + configure(argument: _grpc_testing_ClientConfigureRequest, options: grpc.CallOptions, callback: (error?: grpc.ServiceError, result?: _grpc_testing_ClientConfigureResponse__Output) => void): grpc.ClientUnaryCall; + configure(argument: _grpc_testing_ClientConfigureRequest, callback: (error?: grpc.ServiceError, result?: _grpc_testing_ClientConfigureResponse__Output) => void): grpc.ClientUnaryCall; + +} + +/** + * A service to dynamically update the configuration of an xDS test client. + */ +export interface XdsUpdateClientConfigureServiceHandlers extends grpc.UntypedServiceImplementation { + /** + * Update the tes client's configuration. + */ + Configure(call: grpc.ServerUnaryCall<_grpc_testing_ClientConfigureRequest__Output, _grpc_testing_ClientConfigureResponse>, callback: grpc.sendUnaryData<_grpc_testing_ClientConfigureResponse>): void; + +} diff --git a/packages/grpc-js-xds/interop/generated/test.ts b/packages/grpc-js-xds/interop/generated/test.ts index 330dbc9f..aedfa245 100644 --- a/packages/grpc-js-xds/interop/generated/test.ts +++ b/packages/grpc-js-xds/interop/generated/test.ts @@ -5,6 +5,7 @@ import { LoadBalancerStatsServiceClient as _grpc_testing_LoadBalancerStatsServic import { ReconnectServiceClient as _grpc_testing_ReconnectServiceClient } from './grpc/testing/ReconnectService'; import { TestServiceClient as _grpc_testing_TestServiceClient } from './grpc/testing/TestService'; import { UnimplementedServiceClient as _grpc_testing_UnimplementedServiceClient } from './grpc/testing/UnimplementedService'; +import { XdsUpdateClientConfigureServiceClient as _grpc_testing_XdsUpdateClientConfigureServiceClient } from './grpc/testing/XdsUpdateClientConfigureService'; import { XdsUpdateHealthServiceClient as _grpc_testing_XdsUpdateHealthServiceClient } from './grpc/testing/XdsUpdateHealthService'; type ConstructorArguments = Constructor extends new (...args: infer Args) => any ? Args: never; @@ -16,9 +17,13 @@ export interface ProtoGrpcType { grpc: { testing: { BoolValue: MessageTypeDefinition + ClientConfigureRequest: MessageTypeDefinition + ClientConfigureResponse: MessageTypeDefinition EchoStatus: MessageTypeDefinition Empty: MessageTypeDefinition GrpclbRouteType: EnumTypeDefinition + LoadBalancerAccumulatedStatsRequest: MessageTypeDefinition + LoadBalancerAccumulatedStatsResponse: MessageTypeDefinition LoadBalancerStatsRequest: MessageTypeDefinition LoadBalancerStatsResponse: MessageTypeDefinition /** @@ -50,6 +55,10 @@ export interface ProtoGrpcType { * that case. */ UnimplementedService: SubtypeConstructor & { service: ServiceDefinition } + /** + * A service to dynamically update the configuration of an xDS test client. + */ + XdsUpdateClientConfigureService: SubtypeConstructor & { service: ServiceDefinition } /** * A service to remotely control health status of an xDS test server. */ diff --git a/packages/grpc-js-xds/interop/xds-interop-client.ts b/packages/grpc-js-xds/interop/xds-interop-client.ts index 526c5194..c3cb55d5 100644 --- a/packages/grpc-js-xds/interop/xds-interop-client.ts +++ b/packages/grpc-js-xds/interop/xds-interop-client.ts @@ -26,6 +26,9 @@ import { TestServiceClient } from './generated/grpc/testing/TestService'; import { LoadBalancerStatsResponse } from './generated/grpc/testing/LoadBalancerStatsResponse'; import * as yargs from 'yargs'; import { LoadBalancerStatsServiceHandlers } from './generated/grpc/testing/LoadBalancerStatsService'; +import { XdsUpdateClientConfigureServiceHandlers } from './generated/grpc/testing/XdsUpdateClientConfigureService'; +import { Empty__Output } from './generated/grpc/testing/Empty'; +import { LoadBalancerAccumulatedStatsResponse } from './generated/grpc/testing/LoadBalancerAccumulatedStatsResponse'; grpc_xds.register(); @@ -159,47 +162,95 @@ class CallStatsTracker { } } -function sendConstantQps(client: TestServiceClient, qps: number, failOnFailedRpcs: boolean, callStatsTracker: CallStatsTracker) { - let anyCallSucceeded: boolean = false; - setInterval(() => { - const notifier = callStatsTracker.startCall(); - let gotMetadata: boolean = false; - let hostname: string | null = null; - let completed: boolean = false; - let completedWithError: boolean = false; - const deadline = new Date(); - deadline.setSeconds(deadline.getSeconds() + REQUEST_TIMEOUT_SEC); - const call = client.emptyCall({}, {deadline}, (error, value) => { - if (error) { - if (failOnFailedRpcs && anyCallSucceeded) { - console.error('A call failed after a call succeeded'); - process.exit(1); - } - completed = true; - completedWithError = true; - notifier.onCallFailed(error.message); - } else { - anyCallSucceeded = true; - if (gotMetadata) { - if (hostname === null) { - notifier.onCallFailed('Hostname omitted from call metadata'); - } else { - notifier.onCallSucceeded(hostname); - } - } +type CallType = 'EMPTY_CALL' | 'UNARY_CALL'; + +interface ClientConfiguration { + callTypes: (CallType)[]; + metadata: { + EMPTY_CALL: grpc.Metadata, + UNARY_CALL: grpc.Metadata + }, + timeoutSec: number +} + +const currentConfig: ClientConfiguration = { + callTypes: ['EMPTY_CALL'], + metadata: { + EMPTY_CALL: new grpc.Metadata(), + UNARY_CALL: new grpc.Metadata() + }, + timeoutSec: REQUEST_TIMEOUT_SEC +}; + +let anyCallSucceeded = false; + +const accumulatedStats: LoadBalancerAccumulatedStatsResponse = { + stats_per_method: { + 'EMPTY_CALL': { + rpcs_started: 0, + result: {} + }, + 'UNARY_CALL': { + rpcs_started: 0, + result: {} + } + } +}; + +function makeSingleRequest(client: TestServiceClient, type: CallType, failOnFailedRpcs: boolean, callStatsTracker: CallStatsTracker) { + const callTypeStats = accumulatedStats.stats_per_method![type]; + callTypeStats.rpcs_started! += 1; + + const notifier = callStatsTracker.startCall(); + let gotMetadata: boolean = false; + let hostname: string | null = null; + let completed: boolean = false; + let completedWithError: boolean = false; + const deadline = new Date(); + deadline.setSeconds(deadline.getSeconds() + currentConfig.timeoutSec); + const callback = (error: grpc.ServiceError | undefined, value: Empty__Output | undefined) => { + const statusCode = error?.code ?? grpc.status.OK; + callTypeStats.result![statusCode] = (callTypeStats.result![statusCode] ?? 0) + 1; + if (error) { + if (failOnFailedRpcs && anyCallSucceeded) { + console.error('A call failed after a call succeeded'); + process.exit(1); } - }); - call.on('metadata', (metadata) => { - hostname = (metadata.get('hostname') as string[])[0] ?? null; - gotMetadata = true; - if (completed && !completedWithError) { + completed = true; + completedWithError = true; + notifier.onCallFailed(error.message); + } else { + anyCallSucceeded = true; + if (gotMetadata) { if (hostname === null) { notifier.onCallFailed('Hostname omitted from call metadata'); } else { notifier.onCallSucceeded(hostname); } } - }) + } + }; + const method = (type === 'EMPTY_CALL' ? client.emptyCall : client.unaryCall).bind(client); + const call = method({}, currentConfig.metadata[type], {deadline}, callback); + call.on('metadata', (metadata) => { + hostname = (metadata.get('hostname') as string[])[0] ?? null; + gotMetadata = true; + if (completed && !completedWithError) { + if (hostname === null) { + notifier.onCallFailed('Hostname omitted from call metadata'); + } else { + notifier.onCallSucceeded(hostname); + } + } + }); + +} + +function sendConstantQps(client: TestServiceClient, qps: number, failOnFailedRpcs: boolean, callStatsTracker: CallStatsTracker) { + setInterval(() => { + for (const callType of currentConfig.callTypes) { + makeSingleRequest(client, callType, failOnFailedRpcs, callStatsTracker); + } }, 1000/qps); } @@ -234,11 +285,30 @@ function main() { }, (error) => { callback({code: grpc.status.ABORTED, details: 'Call stats collection failed'}); }); + }, + GetClientAccumulatedStats: (call, callback) => { + callback(null, accumulatedStats); + } + } + + const xdsUpdateClientConfigureServiceImpl: XdsUpdateClientConfigureServiceHandlers = { + Configure: (call, callback) => { + const callMetadata = { + EMPTY_CALL: new grpc.Metadata(), + UNARY_CALL: new grpc.Metadata() + } + for (const metadataItem of call.request.metadata) { + callMetadata[metadataItem.type].add(metadataItem.key, metadataItem.value); + } + currentConfig.callTypes = call.request.types; + currentConfig.metadata = callMetadata; + currentConfig.timeoutSec = call.request.timeout_sec } } const server = new grpc.Server(); server.addService(loadedProto.grpc.testing.LoadBalancerStatsService.service, loadBalancerStatsServiceImpl); + server.addService(loadedProto.grpc.testing.XdsUpdateClientConfigureService.service, xdsUpdateClientConfigureServiceImpl); server.bindAsync(`0.0.0.0:${argv.stats_port}`, grpc.ServerCredentials.createInsecure(), (error, port) => { if (error) { throw error; diff --git a/packages/grpc-js-xds/proto/grpc/testing/messages.proto b/packages/grpc-js-xds/proto/grpc/testing/messages.proto index 70e34277..559876ed 100644 --- a/packages/grpc-js-xds/proto/grpc/testing/messages.proto +++ b/packages/grpc-js-xds/proto/grpc/testing/messages.proto @@ -212,3 +212,59 @@ message LoadBalancerStatsResponse { int32 num_failures = 2; map rpcs_by_method = 3; } + +// Request for retrieving a test client's accumulated stats. +message LoadBalancerAccumulatedStatsRequest {} + +// Accumulated stats for RPCs sent by a test client. +message LoadBalancerAccumulatedStatsResponse { + // The total number of RPCs have ever issued for each type. + // Deprecated: use stats_per_method.rpcs_started instead. + map num_rpcs_started_by_method = 1 [deprecated = true]; + // The total number of RPCs have ever completed successfully for each type. + // Deprecated: use stats_per_method.result instead. + map num_rpcs_succeeded_by_method = 2 [deprecated = true]; + // The total number of RPCs have ever failed for each type. + // Deprecated: use stats_per_method.result instead. + map num_rpcs_failed_by_method = 3 [deprecated = true]; + + message MethodStats { + // The number of RPCs that were started for this method. + int32 rpcs_started = 1; + + // The number of RPCs that completed with each status for this method. The + // key is the integral value of a google.rpc.Code; the value is the count. + map result = 2; + } + + // Per-method RPC statistics. The key is the RpcType in string form; e.g. + // 'EMPTY_CALL' or 'UNARY_CALL' + map stats_per_method = 4; +} + +// Configurations for a test client. +message ClientConfigureRequest { + // Type of RPCs to send. + enum RpcType { + EMPTY_CALL = 0; + UNARY_CALL = 1; + } + + // Metadata to be attached for the given type of RPCs. + message Metadata { + RpcType type = 1; + string key = 2; + string value = 3; + } + + // The types of RPCs the client sends. + repeated RpcType types = 1; + // The collection of custom metadata to be attached to RPCs sent by the client. + repeated Metadata metadata = 2; + // The deadline to use, in seconds, for all RPCs. If unset or zero, the + // client will use the default from the command-line. + int32 timeout_sec = 3; +} + +// Response for updating a test client's configuration. +message ClientConfigureResponse {} diff --git a/packages/grpc-js-xds/proto/grpc/testing/test.proto b/packages/grpc-js-xds/proto/grpc/testing/test.proto index 9d0fadd9..b2606a00 100644 --- a/packages/grpc-js-xds/proto/grpc/testing/test.proto +++ b/packages/grpc-js-xds/proto/grpc/testing/test.proto @@ -83,6 +83,10 @@ service LoadBalancerStatsService { // Gets the backend distribution for RPCs sent by a test client. rpc GetClientStats(LoadBalancerStatsRequest) returns (LoadBalancerStatsResponse) {} + + // Gets the accumulated stats for RPCs sent by a test client. + rpc GetClientAccumulatedStats(LoadBalancerAccumulatedStatsRequest) + returns (LoadBalancerAccumulatedStatsResponse) {} } // A service to remotely control health status of an xDS test server. @@ -90,3 +94,9 @@ service XdsUpdateHealthService { rpc SetServing(grpc.testing.Empty) returns (grpc.testing.Empty); rpc SetNotServing(grpc.testing.Empty) returns (grpc.testing.Empty); } + +// A service to dynamically update the configuration of an xDS test client. +service XdsUpdateClientConfigureService { + // Update the tes client's configuration. + rpc Configure(ClientConfigureRequest) returns (ClientConfigureResponse); +} From 65a16397981655293c9593df5c26d99c65d24db2 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Tue, 16 Mar 2021 13:43:53 -0700 Subject: [PATCH 39/46] grpc-tools: Bump protobuf dependency to 3.15.6 --- packages/grpc-tools/deps/protobuf | 2 +- packages/grpc-tools/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/grpc-tools/deps/protobuf b/packages/grpc-tools/deps/protobuf index 2514f0bd..6aa539bf 160000 --- a/packages/grpc-tools/deps/protobuf +++ b/packages/grpc-tools/deps/protobuf @@ -1 +1 @@ -Subproject commit 2514f0bd7da7e2af1bed4c5d1b84f031c4d12c10 +Subproject commit 6aa539bf0195f188ff86efe6fb8bfa2b676cdd46 diff --git a/packages/grpc-tools/package.json b/packages/grpc-tools/package.json index 86f45dc3..5c7337f6 100644 --- a/packages/grpc-tools/package.json +++ b/packages/grpc-tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.10.0", + "version": "1.11.0", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "https://grpc.io/", From 550a4e93f52f3fc028a50d55110048cf4195f833 Mon Sep 17 00:00:00 2001 From: sovlookup Date: Wed, 17 Mar 2021 09:32:02 +0800 Subject: [PATCH 40/46] proto-loader: update fromJSON --- packages/proto-loader/src/index.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/proto-loader/src/index.ts b/packages/proto-loader/src/index.ts index 237fd840..883f7d60 100644 --- a/packages/proto-loader/src/index.ts +++ b/packages/proto-loader/src/index.ts @@ -399,17 +399,10 @@ export function fromJSON( json: Protobuf.INamespace, root?: Protobuf.Root ): PackageDefinition { - const options: Options = json.options || {}; const newRoot: Protobuf.Root = root || new Protobuf.Root(); - if (!!options.includeDirs) { - if (!Array.isArray(options.includeDirs)) { - throw new Error('The includeDirs option must be an array'); - } - addIncludePathResolver(newRoot, options.includeDirs as string[]); - } const loadedRoot = Protobuf.Root.fromJSON(json, newRoot); loadedRoot.resolveAll(); - return createPackageDefinition(newRoot, options!); + return createPackageDefinition(newRoot, {}); } export function loadFileDescriptorSetFromBuffer( From b7bf2bf6cd26551d99ee7d7982031b1503d2dd6b Mon Sep 17 00:00:00 2001 From: sovlookup Date: Wed, 17 Mar 2021 09:41:22 +0800 Subject: [PATCH 41/46] proto-loader: update fromJSON remove Protobuf.Root 'json' param --- packages/proto-loader/src/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/proto-loader/src/index.ts b/packages/proto-loader/src/index.ts index 883f7d60..69c21ecd 100644 --- a/packages/proto-loader/src/index.ts +++ b/packages/proto-loader/src/index.ts @@ -396,10 +396,9 @@ export function loadSync( } export function fromJSON( - json: Protobuf.INamespace, - root?: Protobuf.Root + json: Protobuf.INamespace ): PackageDefinition { - const newRoot: Protobuf.Root = root || new Protobuf.Root(); + const newRoot: Protobuf.Root = new Protobuf.Root(); const loadedRoot = Protobuf.Root.fromJSON(json, newRoot); loadedRoot.resolveAll(); return createPackageDefinition(newRoot, {}); From fe2e93d30a3775b686be8de917f7a51b8da52e44 Mon Sep 17 00:00:00 2001 From: sovlookup Date: Wed, 17 Mar 2021 09:48:09 +0800 Subject: [PATCH 42/46] proto-loader: update fromJSON test --- .../proto-loader/test_protos/rpc.proto.json | 1515 ++++++++++++++++- 1 file changed, 1508 insertions(+), 7 deletions(-) diff --git a/packages/proto-loader/test_protos/rpc.proto.json b/packages/proto-loader/test_protos/rpc.proto.json index e49dec1d..5f96dba9 100644 --- a/packages/proto-loader/test_protos/rpc.proto.json +++ b/packages/proto-loader/test_protos/rpc.proto.json @@ -1,16 +1,1517 @@ { + "options": { + "java_package": "com.google.apps.jspb.proto", + "java_multiple_files": true + }, "nested": { - "awesomepackage": { + "jspb": { "nested": { - "AwesomeMessage": { - "fields": { - "awesomeField": { - "type": "string", - "id": 1 + "test": { + "nested": { + "Empty": { + "fields": {} + }, + "OuterEnum": { + "values": { + "FOO": 1, + "BAR": 2 + } + }, + "EnumContainer": { + "fields": { + "outerEnum": { + "type": "OuterEnum", + "id": 1 + } + } + }, + "Simple1": { + "fields": { + "aString": { + "rule": "required", + "type": "string", + "id": 1 + }, + "aRepeatedString": { + "rule": "repeated", + "type": "string", + "id": 2 + }, + "aBoolean": { + "type": "bool", + "id": 3 + } + } + }, + "Simple2": { + "fields": { + "aString": { + "rule": "required", + "type": "string", + "id": 1 + }, + "aRepeatedString": { + "rule": "repeated", + "type": "string", + "id": 2 + } + } + }, + "SpecialCases": { + "fields": { + "normal": { + "rule": "required", + "type": "string", + "id": 1 + }, + "default": { + "rule": "required", + "type": "string", + "id": 2 + }, + "function": { + "rule": "required", + "type": "string", + "id": 3 + }, + "var": { + "rule": "required", + "type": "string", + "id": 4 + } + } + }, + "OptionalFields": { + "fields": { + "aString": { + "type": "string", + "id": 1 + }, + "aBool": { + "rule": "required", + "type": "bool", + "id": 2 + }, + "aNestedMessage": { + "type": "Nested", + "id": 3 + }, + "aRepeatedMessage": { + "rule": "repeated", + "type": "Nested", + "id": 4 + }, + "aRepeatedString": { + "rule": "repeated", + "type": "string", + "id": 5 + } + }, + "nested": { + "Nested": { + "fields": { + "anInt": { + "type": "int32", + "id": 1 + } + } + } + } + }, + "HasExtensions": { + "fields": { + "str1": { + "type": "string", + "id": 1 + }, + "str2": { + "type": "string", + "id": 2 + }, + "str3": { + "type": "string", + "id": 3 + } + }, + "extensions": [ + [ + 10, + 536870911 + ] + ] + }, + "Complex": { + "fields": { + "aString": { + "rule": "required", + "type": "string", + "id": 1 + }, + "anOutOfOrderBool": { + "rule": "required", + "type": "bool", + "id": 9 + }, + "aNestedMessage": { + "type": "Nested", + "id": 4 + }, + "aRepeatedMessage": { + "rule": "repeated", + "type": "Nested", + "id": 5 + }, + "aRepeatedString": { + "rule": "repeated", + "type": "string", + "id": 7 + } + }, + "nested": { + "Nested": { + "fields": { + "anInt": { + "rule": "required", + "type": "int32", + "id": 2 + } + } + } + } + }, + "OuterMessage": { + "fields": {}, + "nested": { + "Complex": { + "fields": { + "innerComplexField": { + "type": "int32", + "id": 1 + } + } + } + } + }, + "IsExtension": { + "fields": { + "ext1": { + "type": "string", + "id": 1 + } + }, + "nested": { + "extField": { + "type": "IsExtension", + "id": 100, + "extend": "HasExtensions" + }, + "simpleOption": { + "type": "string", + "id": 42113038, + "extend": "google.protobuf.EnumOptions" + } + } + }, + "IndirectExtension": { + "fields": {}, + "nested": { + "simple": { + "type": "Simple1", + "id": 101, + "extend": "HasExtensions" + }, + "str": { + "type": "string", + "id": 102, + "extend": "HasExtensions" + }, + "repeatedStr": { + "rule": "repeated", + "type": "string", + "id": 103, + "extend": "HasExtensions" + }, + "repeatedSimple": { + "rule": "repeated", + "type": "Simple1", + "id": 104, + "extend": "HasExtensions" + } + } + }, + "simple1": { + "type": "Simple1", + "id": 105, + "extend": "HasExtensions" + }, + "DefaultValues": { + "fields": { + "stringField": { + "type": "string", + "id": 1, + "options": { + "default": "default<>abc" + } + }, + "boolField": { + "type": "bool", + "id": 2, + "options": { + "default": true + } + }, + "intField": { + "type": "int64", + "id": 3, + "options": { + "default": 11 + } + }, + "enumField": { + "type": "Enum", + "id": 4, + "options": { + "default": "E1" + } + }, + "emptyField": { + "type": "string", + "id": 6, + "options": { + "default": "" + } + }, + "bytesField": { + "type": "bytes", + "id": 8, + "options": { + "default": "moo" + } + } + }, + "nested": { + "Enum": { + "values": { + "E1": 13, + "E2": 77 + } + } + } + }, + "FloatingPointFields": { + "fields": { + "optionalFloatField": { + "type": "float", + "id": 1 + }, + "requiredFloatField": { + "rule": "required", + "type": "float", + "id": 2 + }, + "repeatedFloatField": { + "rule": "repeated", + "type": "float", + "id": 3, + "options": { + "packed": false + } + }, + "defaultFloatField": { + "type": "float", + "id": 4, + "options": { + "default": 2 + } + }, + "optionalDoubleField": { + "type": "double", + "id": 5 + }, + "requiredDoubleField": { + "rule": "required", + "type": "double", + "id": 6 + }, + "repeatedDoubleField": { + "rule": "repeated", + "type": "double", + "id": 7, + "options": { + "packed": false + } + }, + "defaultDoubleField": { + "type": "double", + "id": 8, + "options": { + "default": 2 + } + } + } + }, + "TestClone": { + "fields": { + "str": { + "type": "string", + "id": 1 + }, + "simple1": { + "type": "Simple1", + "id": 3 + }, + "simple2": { + "rule": "repeated", + "type": "Simple1", + "id": 5 + }, + "bytesField": { + "type": "bytes", + "id": 6 + }, + "unused": { + "type": "string", + "id": 7 + } + }, + "extensions": [ + [ + 10, + 536870911 + ] + ] + }, + "CloneExtension": { + "fields": { + "ext": { + "type": "string", + "id": 2 + } + }, + "nested": { + "extField": { + "type": "CloneExtension", + "id": 100, + "extend": "TestClone" + } + } + }, + "TestGroup": { + "fields": { + "repeatedGroup": { + "rule": "repeated", + "type": "RepeatedGroup", + "id": 1 + }, + "requiredGroup": { + "rule": "required", + "type": "RequiredGroup", + "id": 2 + }, + "optionalGroup": { + "type": "OptionalGroup", + "id": 3 + }, + "id": { + "type": "string", + "id": 4 + }, + "requiredSimple": { + "rule": "required", + "type": "Simple2", + "id": 5 + }, + "optionalSimple": { + "type": "Simple2", + "id": 6 + } + }, + "nested": { + "RepeatedGroup": { + "fields": { + "id": { + "rule": "required", + "type": "string", + "id": 1 + }, + "someBool": { + "rule": "repeated", + "type": "bool", + "id": 2, + "options": { + "packed": false + } + } + }, + "group": true + }, + "RequiredGroup": { + "fields": { + "id": { + "rule": "required", + "type": "string", + "id": 1 + } + }, + "group": true + }, + "OptionalGroup": { + "fields": { + "id": { + "rule": "required", + "type": "string", + "id": 1 + } + }, + "group": true + } + } + }, + "TestGroup1": { + "fields": { + "group": { + "type": "TestGroup.RepeatedGroup", + "id": 1 + } + } + }, + "TestReservedNames": { + "fields": { + "extension": { + "type": "int32", + "id": 1 + } + }, + "extensions": [ + [ + 10, + 536870911 + ] + ] + }, + "TestReservedNamesExtension": { + "fields": {}, + "nested": { + "foo": { + "type": "int32", + "id": 10, + "extend": "TestReservedNames" + } + } + }, + "TestMessageWithOneof": { + "oneofs": { + "partialOneof": { + "oneof": [ + "pone", + "pthree" + ] + }, + "recursiveOneof": { + "oneof": [ + "rone", + "rtwo" + ] + }, + "defaultOneofA": { + "oneof": [ + "aone", + "atwo" + ] + }, + "defaultOneofB": { + "oneof": [ + "bone", + "btwo" + ] + } + }, + "fields": { + "pone": { + "type": "string", + "id": 3 + }, + "pthree": { + "type": "string", + "id": 5 + }, + "rone": { + "type": "TestMessageWithOneof", + "id": 6 + }, + "rtwo": { + "type": "string", + "id": 7 + }, + "normalField": { + "type": "bool", + "id": 8 + }, + "repeatedField": { + "rule": "repeated", + "type": "string", + "id": 9 + }, + "aone": { + "type": "int32", + "id": 10, + "options": { + "default": 1234 + } + }, + "atwo": { + "type": "int32", + "id": 11 + }, + "bone": { + "type": "int32", + "id": 12 + }, + "btwo": { + "type": "int32", + "id": 13, + "options": { + "default": 1234 + } + } + } + }, + "TestEndsWithBytes": { + "fields": { + "value": { + "type": "int32", + "id": 1 + }, + "data": { + "type": "bytes", + "id": 2 + } + } + }, + "TestMapFieldsNoBinary": { + "fields": { + "mapStringString": { + "keyType": "string", + "type": "string", + "id": 1 + }, + "mapStringInt32": { + "keyType": "string", + "type": "int32", + "id": 2 + }, + "mapStringInt64": { + "keyType": "string", + "type": "int64", + "id": 3 + }, + "mapStringBool": { + "keyType": "string", + "type": "bool", + "id": 4 + }, + "mapStringDouble": { + "keyType": "string", + "type": "double", + "id": 5 + }, + "mapStringEnum": { + "keyType": "string", + "type": "MapValueEnumNoBinary", + "id": 6 + }, + "mapStringMsg": { + "keyType": "string", + "type": "MapValueMessageNoBinary", + "id": 7 + }, + "mapInt32String": { + "keyType": "int32", + "type": "string", + "id": 8 + }, + "mapInt64String": { + "keyType": "int64", + "type": "string", + "id": 9 + }, + "mapBoolString": { + "keyType": "bool", + "type": "string", + "id": 10 + }, + "testMapFields": { + "type": "TestMapFieldsNoBinary", + "id": 11 + }, + "mapStringTestmapfields": { + "keyType": "string", + "type": "TestMapFieldsNoBinary", + "id": 12 + } + } + }, + "MapValueEnumNoBinary": { + "values": { + "MAP_VALUE_FOO_NOBINARY": 0, + "MAP_VALUE_BAR_NOBINARY": 1, + "MAP_VALUE_BAZ_NOBINARY": 2 + } + }, + "MapValueMessageNoBinary": { + "fields": { + "foo": { + "type": "int32", + "id": 1 + } + } + }, + "Deeply": { + "fields": {}, + "nested": { + "Nested": { + "fields": {}, + "nested": { + "Message": { + "fields": { + "count": { + "type": "int32", + "id": 1 + } + } + } + } + } + } + } + } + } + } + }, + "google": { + "nested": { + "protobuf": { + "options": { + "go_package": "descriptor", + "java_package": "com.google.protobuf", + "java_outer_classname": "DescriptorProtos", + "csharp_namespace": "Google.Protobuf.Reflection", + "objc_class_prefix": "GPB", + "optimize_for": "SPEED" + }, + "nested": { + "FileDescriptorSet": { + "fields": { + "file": { + "rule": "repeated", + "type": "FileDescriptorProto", + "id": 1 + } + } + }, + "FileDescriptorProto": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "package": { + "type": "string", + "id": 2 + }, + "dependency": { + "rule": "repeated", + "type": "string", + "id": 3 + }, + "publicDependency": { + "rule": "repeated", + "type": "int32", + "id": 10, + "options": { + "packed": false + } + }, + "weakDependency": { + "rule": "repeated", + "type": "int32", + "id": 11, + "options": { + "packed": false + } + }, + "messageType": { + "rule": "repeated", + "type": "DescriptorProto", + "id": 4 + }, + "enumType": { + "rule": "repeated", + "type": "EnumDescriptorProto", + "id": 5 + }, + "service": { + "rule": "repeated", + "type": "ServiceDescriptorProto", + "id": 6 + }, + "extension": { + "rule": "repeated", + "type": "FieldDescriptorProto", + "id": 7 + }, + "options": { + "type": "FileOptions", + "id": 8 + }, + "sourceCodeInfo": { + "type": "SourceCodeInfo", + "id": 9 + }, + "syntax": { + "type": "string", + "id": 12 + } + } + }, + "DescriptorProto": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "field": { + "rule": "repeated", + "type": "FieldDescriptorProto", + "id": 2 + }, + "extension": { + "rule": "repeated", + "type": "FieldDescriptorProto", + "id": 6 + }, + "nestedType": { + "rule": "repeated", + "type": "DescriptorProto", + "id": 3 + }, + "enumType": { + "rule": "repeated", + "type": "EnumDescriptorProto", + "id": 4 + }, + "extensionRange": { + "rule": "repeated", + "type": "ExtensionRange", + "id": 5 + }, + "oneofDecl": { + "rule": "repeated", + "type": "OneofDescriptorProto", + "id": 8 + }, + "options": { + "type": "MessageOptions", + "id": 7 + }, + "reservedRange": { + "rule": "repeated", + "type": "ReservedRange", + "id": 9 + }, + "reservedName": { + "rule": "repeated", + "type": "string", + "id": 10 + } + }, + "nested": { + "ExtensionRange": { + "fields": { + "start": { + "type": "int32", + "id": 1 + }, + "end": { + "type": "int32", + "id": 2 + } + } + }, + "ReservedRange": { + "fields": { + "start": { + "type": "int32", + "id": 1 + }, + "end": { + "type": "int32", + "id": 2 + } + } + } + } + }, + "FieldDescriptorProto": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "number": { + "type": "int32", + "id": 3 + }, + "label": { + "type": "Label", + "id": 4 + }, + "type": { + "type": "Type", + "id": 5 + }, + "typeName": { + "type": "string", + "id": 6 + }, + "extendee": { + "type": "string", + "id": 2 + }, + "defaultValue": { + "type": "string", + "id": 7 + }, + "oneofIndex": { + "type": "int32", + "id": 9 + }, + "jsonName": { + "type": "string", + "id": 10 + }, + "options": { + "type": "FieldOptions", + "id": 8 + } + }, + "nested": { + "Type": { + "values": { + "TYPE_DOUBLE": 1, + "TYPE_FLOAT": 2, + "TYPE_INT64": 3, + "TYPE_UINT64": 4, + "TYPE_INT32": 5, + "TYPE_FIXED64": 6, + "TYPE_FIXED32": 7, + "TYPE_BOOL": 8, + "TYPE_STRING": 9, + "TYPE_GROUP": 10, + "TYPE_MESSAGE": 11, + "TYPE_BYTES": 12, + "TYPE_UINT32": 13, + "TYPE_ENUM": 14, + "TYPE_SFIXED32": 15, + "TYPE_SFIXED64": 16, + "TYPE_SINT32": 17, + "TYPE_SINT64": 18 + } + }, + "Label": { + "values": { + "LABEL_OPTIONAL": 1, + "LABEL_REQUIRED": 2, + "LABEL_REPEATED": 3 + } + } + } + }, + "OneofDescriptorProto": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "options": { + "type": "OneofOptions", + "id": 2 + } + } + }, + "EnumDescriptorProto": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "value": { + "rule": "repeated", + "type": "EnumValueDescriptorProto", + "id": 2 + }, + "options": { + "type": "EnumOptions", + "id": 3 + } + } + }, + "EnumValueDescriptorProto": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "number": { + "type": "int32", + "id": 2 + }, + "options": { + "type": "EnumValueOptions", + "id": 3 + } + } + }, + "ServiceDescriptorProto": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "method": { + "rule": "repeated", + "type": "MethodDescriptorProto", + "id": 2 + }, + "options": { + "type": "ServiceOptions", + "id": 3 + } + } + }, + "MethodDescriptorProto": { + "fields": { + "name": { + "type": "string", + "id": 1 + }, + "inputType": { + "type": "string", + "id": 2 + }, + "outputType": { + "type": "string", + "id": 3 + }, + "options": { + "type": "MethodOptions", + "id": 4 + }, + "clientStreaming": { + "type": "bool", + "id": 5, + "options": { + "default": false + } + }, + "serverStreaming": { + "type": "bool", + "id": 6, + "options": { + "default": false + } + } + } + }, + "FileOptions": { + "fields": { + "javaPackage": { + "type": "string", + "id": 1 + }, + "javaOuterClassname": { + "type": "string", + "id": 8 + }, + "javaMultipleFiles": { + "type": "bool", + "id": 10, + "options": { + "default": false + } + }, + "javaGenerateEqualsAndHash": { + "type": "bool", + "id": 20, + "options": { + "deprecated": true + } + }, + "javaStringCheckUtf8": { + "type": "bool", + "id": 27, + "options": { + "default": false + } + }, + "optimizeFor": { + "type": "OptimizeMode", + "id": 9, + "options": { + "default": "SPEED" + } + }, + "goPackage": { + "type": "string", + "id": 11 + }, + "ccGenericServices": { + "type": "bool", + "id": 16, + "options": { + "default": false + } + }, + "javaGenericServices": { + "type": "bool", + "id": 17, + "options": { + "default": false + } + }, + "pyGenericServices": { + "type": "bool", + "id": 18, + "options": { + "default": false + } + }, + "deprecated": { + "type": "bool", + "id": 23, + "options": { + "default": false + } + }, + "ccEnableArenas": { + "type": "bool", + "id": 31, + "options": { + "default": false + } + }, + "objcClassPrefix": { + "type": "string", + "id": 36 + }, + "csharpNamespace": { + "type": "string", + "id": 37 + }, + "uninterpretedOption": { + "rule": "repeated", + "type": "UninterpretedOption", + "id": 999 + } + }, + "extensions": [ + [ + 1000, + 536870911 + ] + ], + "reserved": [ + [ + 38, + 38 + ] + ], + "nested": { + "OptimizeMode": { + "values": { + "SPEED": 1, + "CODE_SIZE": 2, + "LITE_RUNTIME": 3 + } + } + } + }, + "MessageOptions": { + "fields": { + "messageSetWireFormat": { + "type": "bool", + "id": 1, + "options": { + "default": false + } + }, + "noStandardDescriptorAccessor": { + "type": "bool", + "id": 2, + "options": { + "default": false + } + }, + "deprecated": { + "type": "bool", + "id": 3, + "options": { + "default": false + } + }, + "mapEntry": { + "type": "bool", + "id": 7 + }, + "uninterpretedOption": { + "rule": "repeated", + "type": "UninterpretedOption", + "id": 999 + } + }, + "extensions": [ + [ + 1000, + 536870911 + ] + ], + "reserved": [ + [ + 8, + 8 + ] + ] + }, + "FieldOptions": { + "fields": { + "ctype": { + "type": "CType", + "id": 1, + "options": { + "default": "STRING" + } + }, + "packed": { + "type": "bool", + "id": 2 + }, + "jstype": { + "type": "JSType", + "id": 6, + "options": { + "default": "JS_NORMAL" + } + }, + "lazy": { + "type": "bool", + "id": 5, + "options": { + "default": false + } + }, + "deprecated": { + "type": "bool", + "id": 3, + "options": { + "default": false + } + }, + "weak": { + "type": "bool", + "id": 10, + "options": { + "default": false + } + }, + "uninterpretedOption": { + "rule": "repeated", + "type": "UninterpretedOption", + "id": 999 + } + }, + "extensions": [ + [ + 1000, + 536870911 + ] + ], + "reserved": [ + [ + 4, + 4 + ] + ], + "nested": { + "CType": { + "values": { + "STRING": 0, + "CORD": 1, + "STRING_PIECE": 2 + } + }, + "JSType": { + "values": { + "JS_NORMAL": 0, + "JS_STRING": 1, + "JS_NUMBER": 2 + } + } + } + }, + "OneofOptions": { + "fields": { + "uninterpretedOption": { + "rule": "repeated", + "type": "UninterpretedOption", + "id": 999 + } + }, + "extensions": [ + [ + 1000, + 536870911 + ] + ] + }, + "EnumOptions": { + "fields": { + "allowAlias": { + "type": "bool", + "id": 2 + }, + "deprecated": { + "type": "bool", + "id": 3, + "options": { + "default": false + } + }, + "uninterpretedOption": { + "rule": "repeated", + "type": "UninterpretedOption", + "id": 999 + } + }, + "extensions": [ + [ + 1000, + 536870911 + ] + ] + }, + "EnumValueOptions": { + "fields": { + "deprecated": { + "type": "bool", + "id": 1, + "options": { + "default": false + } + }, + "uninterpretedOption": { + "rule": "repeated", + "type": "UninterpretedOption", + "id": 999 + } + }, + "extensions": [ + [ + 1000, + 536870911 + ] + ] + }, + "ServiceOptions": { + "fields": { + "deprecated": { + "type": "bool", + "id": 33, + "options": { + "default": false + } + }, + "uninterpretedOption": { + "rule": "repeated", + "type": "UninterpretedOption", + "id": 999 + } + }, + "extensions": [ + [ + 1000, + 536870911 + ] + ] + }, + "MethodOptions": { + "fields": { + "deprecated": { + "type": "bool", + "id": 33, + "options": { + "default": false + } + }, + "idempotencyLevel": { + "type": "IdempotencyLevel", + "id": 34, + "options": { + "default": "IDEMPOTENCY_UNKNOWN" + } + }, + "uninterpretedOption": { + "rule": "repeated", + "type": "UninterpretedOption", + "id": 999 + } + }, + "extensions": [ + [ + 1000, + 536870911 + ] + ], + "nested": { + "IdempotencyLevel": { + "values": { + "IDEMPOTENCY_UNKNOWN": 0, + "NO_SIDE_EFFECTS": 1, + "IDEMPOTENT": 2 + } + } + } + }, + "UninterpretedOption": { + "fields": { + "name": { + "rule": "repeated", + "type": "NamePart", + "id": 2 + }, + "identifierValue": { + "type": "string", + "id": 3 + }, + "positiveIntValue": { + "type": "uint64", + "id": 4 + }, + "negativeIntValue": { + "type": "int64", + "id": 5 + }, + "doubleValue": { + "type": "double", + "id": 6 + }, + "stringValue": { + "type": "bytes", + "id": 7 + }, + "aggregateValue": { + "type": "string", + "id": 8 + } + }, + "nested": { + "NamePart": { + "fields": { + "namePart": { + "rule": "required", + "type": "string", + "id": 1 + }, + "isExtension": { + "rule": "required", + "type": "bool", + "id": 2 + } + } + } + } + }, + "SourceCodeInfo": { + "fields": { + "location": { + "rule": "repeated", + "type": "Location", + "id": 1 + } + }, + "nested": { + "Location": { + "fields": { + "path": { + "rule": "repeated", + "type": "int32", + "id": 1, + "options": { + "packed": true + } + }, + "span": { + "rule": "repeated", + "type": "int32", + "id": 2, + "options": { + "packed": true + } + }, + "leadingComments": { + "type": "string", + "id": 3 + }, + "trailingComments": { + "type": "string", + "id": 4 + }, + "leadingDetachedComments": { + "rule": "repeated", + "type": "string", + "id": 6 + } + } + } + } + }, + "GeneratedCodeInfo": { + "fields": { + "annotation": { + "rule": "repeated", + "type": "Annotation", + "id": 1 + } + }, + "nested": { + "Annotation": { + "fields": { + "path": { + "rule": "repeated", + "type": "int32", + "id": 1, + "options": { + "packed": true + } + }, + "sourceFile": { + "type": "string", + "id": 2 + }, + "begin": { + "type": "int32", + "id": 3 + }, + "end": { + "type": "int32", + "id": 4 + } + } + } + } } } } } } } -} \ No newline at end of file +} From bdd8e1a110a5ac6b1e381395367e2a6d1394e18c Mon Sep 17 00:00:00 2001 From: sovlookup Date: Thu, 18 Mar 2021 15:39:24 +0800 Subject: [PATCH 43/46] proto-loader: fromJSON rm newRoot --- packages/proto-loader/src/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/proto-loader/src/index.ts b/packages/proto-loader/src/index.ts index 69c21ecd..2e827a80 100644 --- a/packages/proto-loader/src/index.ts +++ b/packages/proto-loader/src/index.ts @@ -398,10 +398,9 @@ export function loadSync( export function fromJSON( json: Protobuf.INamespace ): PackageDefinition { - const newRoot: Protobuf.Root = new Protobuf.Root(); - const loadedRoot = Protobuf.Root.fromJSON(json, newRoot); + const loadedRoot = Protobuf.Root.fromJSON(json); loadedRoot.resolveAll(); - return createPackageDefinition(newRoot, {}); + return createPackageDefinition(loadedRoot, {}); } export function loadFileDescriptorSetFromBuffer( From 114386768213345a5d1731e00aed09172464a90e Mon Sep 17 00:00:00 2001 From: sovlookup Date: Thu, 18 Mar 2021 16:21:55 +0800 Subject: [PATCH 44/46] proto-loader: fromJSON add options --- packages/proto-loader/src/index.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/proto-loader/src/index.ts b/packages/proto-loader/src/index.ts index 2e827a80..4cbcff8b 100644 --- a/packages/proto-loader/src/index.ts +++ b/packages/proto-loader/src/index.ts @@ -396,11 +396,16 @@ export function loadSync( } export function fromJSON( - json: Protobuf.INamespace + json: Protobuf.INamespace, + options?: Options ): PackageDefinition { + options = options || {}; + if (!!options.includeDirs) { + throw new Error('The fromJSON does not need to load any files, checkout your options'); + } const loadedRoot = Protobuf.Root.fromJSON(json); loadedRoot.resolveAll(); - return createPackageDefinition(loadedRoot, {}); + return createPackageDefinition(loadedRoot, options!); } export function loadFileDescriptorSetFromBuffer( From 2ce608e1f42a46f37e234343084ac7e563a77265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=8E=E5=8C=97?= <805408477@qq.com> Date: Fri, 19 Mar 2021 08:33:23 +0800 Subject: [PATCH 45/46] Porto-loader fromJSON rm if optiondir --- packages/proto-loader/src/index.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/proto-loader/src/index.ts b/packages/proto-loader/src/index.ts index 4cbcff8b..d60d2bd0 100644 --- a/packages/proto-loader/src/index.ts +++ b/packages/proto-loader/src/index.ts @@ -400,9 +400,6 @@ export function fromJSON( options?: Options ): PackageDefinition { options = options || {}; - if (!!options.includeDirs) { - throw new Error('The fromJSON does not need to load any files, checkout your options'); - } const loadedRoot = Protobuf.Root.fromJSON(json); loadedRoot.resolveAll(); return createPackageDefinition(loadedRoot, options!); From 661fae88c62c46dc35c0333667675ae872a22ba7 Mon Sep 17 00:00:00 2001 From: Michael Lumish Date: Mon, 22 Mar 2021 11:40:23 -0700 Subject: [PATCH 46/46] grpc-tools: make the plugin compatible with proto3 optional fields --- packages/grpc-tools/package.json | 2 +- packages/grpc-tools/src/node_plugin.cc | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/grpc-tools/package.json b/packages/grpc-tools/package.json index 5c7337f6..b1f83ce4 100644 --- a/packages/grpc-tools/package.json +++ b/packages/grpc-tools/package.json @@ -1,6 +1,6 @@ { "name": "grpc-tools", - "version": "1.11.0", + "version": "1.11.1", "author": "Google Inc.", "description": "Tools for developing with gRPC on Node.js", "homepage": "https://grpc.io/", diff --git a/packages/grpc-tools/src/node_plugin.cc b/packages/grpc-tools/src/node_plugin.cc index 9f83578b..b847bcab 100644 --- a/packages/grpc-tools/src/node_plugin.cc +++ b/packages/grpc-tools/src/node_plugin.cc @@ -65,6 +65,10 @@ class NodeGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { coded_out.WriteRaw(code.data(), code.size()); return true; } + + uint64_t GetSupportedFeatures() const override { + return FEATURE_PROTO3_OPTIONAL; + } }; int main(int argc, char* argv[]) {