diff --git a/packages/grpc-js-xds/src/environment.ts b/packages/grpc-js-xds/src/environment.ts index e1c2d815..b8b518da 100644 --- a/packages/grpc-js-xds/src/environment.ts +++ b/packages/grpc-js-xds/src/environment.ts @@ -15,4 +15,5 @@ * */ -export const EXPERIMENTAL_FAULT_INJECTION = (process.env.GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION ?? 'true') === 'true'; \ No newline at end of file +export const EXPERIMENTAL_FAULT_INJECTION = (process.env.GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION ?? 'true') === 'true'; +export const EXPERIMENTAL_OUTLIER_DETECTION = process.env.GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION === 'true'; \ No newline at end of file diff --git a/packages/grpc-js-xds/src/load-balancer-cds.ts b/packages/grpc-js-xds/src/load-balancer-cds.ts index 4829edd4..a6dbf42f 100644 --- a/packages/grpc-js-xds/src/load-balancer-cds.ts +++ b/packages/grpc-js-xds/src/load-balancer-cds.ts @@ -25,8 +25,14 @@ import LoadBalancer = experimental.LoadBalancer; import ChannelControlHelper = experimental.ChannelControlHelper; import registerLoadBalancerType = experimental.registerLoadBalancerType; import LoadBalancingConfig = experimental.LoadBalancingConfig; +import OutlierDetectionLoadBalancingConfig = experimental.OutlierDetectionLoadBalancingConfig; +import SuccessRateEjectionConfig = experimental.SuccessRateEjectionConfig; +import FailurePercentageEjectionConfig = experimental.FailurePercentageEjectionConfig; import { EdsLoadBalancingConfig } from './load-balancer-eds'; import { Watcher } from './xds-stream-state/xds-stream-state'; +import { OutlierDetection__Output } from './generated/envoy/config/cluster/v3/OutlierDetection'; +import { Duration__Output } from './generated/google/protobuf/Duration'; +import { EXPERIMENTAL_OUTLIER_DETECTION } from './environment'; const TRACER_NAME = 'cds_balancer'; @@ -64,6 +70,52 @@ export class CdsLoadBalancingConfig implements LoadBalancingConfig { } } +function durationToMs(duration: Duration__Output): number { + return (Number(duration.seconds) * 1_000 + duration.nanos / 1_000_000) | 0; +} + +function translateOutlierDetectionConfig(outlierDetection: OutlierDetection__Output | null): OutlierDetectionLoadBalancingConfig | undefined { + if (!EXPERIMENTAL_OUTLIER_DETECTION) { + return undefined; + } + if (!outlierDetection) { + /* No-op outlier detection config, with max possible interval and no + * ejection criteria configured. */ + return new OutlierDetectionLoadBalancingConfig(~(1<<31), null, null, null, null, null, []); + } + let successRateConfig: Partial | null = null; + /* Success rate ejection is enabled by default, so we only disable it if + * enforcing_success_rate is set and it has the value 0 */ + if (!outlierDetection.enforcing_success_rate || outlierDetection.enforcing_success_rate.value > 0) { + successRateConfig = { + enforcement_percentage: outlierDetection.enforcing_success_rate?.value, + minimum_hosts: outlierDetection.success_rate_minimum_hosts?.value, + request_volume: outlierDetection.success_rate_request_volume?.value, + stdev_factor: outlierDetection.success_rate_stdev_factor?.value + }; + } + let failurePercentageConfig: Partial | null = null; + /* Failure percentage ejection is disabled by default, so we only enable it + * if enforcing_failure_percentage is set and it has a value greater than 0 */ + if (outlierDetection.enforcing_failure_percentage && outlierDetection.enforcing_failure_percentage.value > 0) { + failurePercentageConfig = { + enforcement_percentage: outlierDetection.enforcing_failure_percentage.value, + minimum_hosts: outlierDetection.failure_percentage_minimum_hosts?.value, + request_volume: outlierDetection.failure_percentage_request_volume?.value, + threshold: outlierDetection.failure_percentage_threshold?.value + } + } + return new OutlierDetectionLoadBalancingConfig( + outlierDetection.interval ? durationToMs(outlierDetection.interval) : null, + outlierDetection.base_ejection_time ? durationToMs(outlierDetection.base_ejection_time) : null, + outlierDetection.max_ejection_time ? durationToMs(outlierDetection.max_ejection_time) : null, + outlierDetection.max_ejection_percent?.value ?? null, + successRateConfig, + failurePercentageConfig, + [] + ); +} + export class CdsLoadBalancer implements LoadBalancer { private childBalancer: ChildLoadBalancerHandler; private watcher: Watcher; @@ -90,7 +142,15 @@ export class CdsLoadBalancer implements LoadBalancer { * used for load reporting as for other xDS operations. Setting * lrsLoadReportingServerName to the empty string sets that behavior. * Otherwise, if the field is omitted, load reporting is disabled. */ - const edsConfig: EdsLoadBalancingConfig = new EdsLoadBalancingConfig(update.name, [], [], update.eds_cluster_config!.service_name === '' ? undefined : update.eds_cluster_config!.service_name, update.lrs_server?.self ? '' : undefined, maxConcurrentRequests); + const edsConfig: EdsLoadBalancingConfig = new EdsLoadBalancingConfig( + /* cluster= */ update.name, + /* localityPickingPolicy= */ [], + /* endpointPickingPolicy= */ [], + /* edsServiceName= */ update.eds_cluster_config!.service_name === '' ? undefined : update.eds_cluster_config!.service_name, + /* lrsLoadReportingServerName= */update.lrs_server?.self ? '' : undefined, + /* maxConcurrentRequests= */ maxConcurrentRequests, + /* outlierDetection= */ translateOutlierDetectionConfig(update.outlier_detection) + ); trace('Child update EDS config: ' + JSON.stringify(edsConfig)); this.childBalancer.updateAddressList( [], diff --git a/packages/grpc-js-xds/src/load-balancer-eds.ts b/packages/grpc-js-xds/src/load-balancer-eds.ts index 55ad714a..e7aac057 100644 --- a/packages/grpc-js-xds/src/load-balancer-eds.ts +++ b/packages/grpc-js-xds/src/load-balancer-eds.ts @@ -38,6 +38,8 @@ import Filter = experimental.Filter; import BaseFilter = experimental.BaseFilter; import FilterFactory = experimental.FilterFactory; import CallStream = experimental.CallStream; +import OutlierDetectionLoadBalancingConfig = experimental.OutlierDetectionLoadBalancingConfig; +import { EXPERIMENTAL_OUTLIER_DETECTION } from './environment'; const TRACER_NAME = 'eds_balancer'; @@ -71,12 +73,15 @@ export class EdsLoadBalancingConfig implements LoadBalancingConfig { if (this.lrsLoadReportingServerName !== undefined) { jsonObj.lrs_load_reporting_server_name = this.lrsLoadReportingServerName; } + if (this.outlierDetection !== undefined) { + jsonObj.outlier_detection = this.outlierDetection.toJsonObject(); + } return { [TYPE_NAME]: jsonObj }; } - constructor(private cluster: string, private localityPickingPolicy: LoadBalancingConfig[], private endpointPickingPolicy: LoadBalancingConfig[], private edsServiceName?: string, private lrsLoadReportingServerName?: string, maxConcurrentRequests?: number) { + constructor(private cluster: string, private localityPickingPolicy: LoadBalancingConfig[], private endpointPickingPolicy: LoadBalancingConfig[], private edsServiceName?: string, private lrsLoadReportingServerName?: string, maxConcurrentRequests?: number, private outlierDetection?: OutlierDetectionLoadBalancingConfig) { this.maxConcurrentRequests = maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS; } @@ -104,6 +109,10 @@ export class EdsLoadBalancingConfig implements LoadBalancingConfig { return this.maxConcurrentRequests; } + getOutlierDetection() { + return this.outlierDetection; + } + static createFromJson(obj: any): EdsLoadBalancingConfig { if (!('cluster' in obj && typeof obj.cluster === 'string')) { throw new Error('eds config must have a string field cluster'); @@ -123,7 +132,17 @@ export class EdsLoadBalancingConfig implements LoadBalancingConfig { if ('max_concurrent_requests' in obj && (!obj.max_concurrent_requests === undefined || typeof obj.max_concurrent_requests === 'number')) { throw new Error('eds config max_concurrent_requests must be a number if provided'); } - return new EdsLoadBalancingConfig(obj.cluster, obj.locality_picking_policy.map(validateLoadBalancingConfig), obj.endpoint_picking_policy.map(validateLoadBalancingConfig), obj.eds_service_name, obj.lrs_load_reporting_server_name, obj.max_concurrent_requests); + let validatedOutlierDetectionConfig: OutlierDetectionLoadBalancingConfig | undefined = undefined; + if (EXPERIMENTAL_OUTLIER_DETECTION) { + if ('outlier_detection' in obj) { + const outlierDetectionConfig = validateLoadBalancingConfig(obj.outlier_detection); + if (!(outlierDetectionConfig instanceof OutlierDetectionLoadBalancingConfig)) { + throw new Error('eds config outlier_detection must be a valid outlier detection config if provided'); + } + validatedOutlierDetectionConfig = outlierDetectionConfig; + } + } + return new EdsLoadBalancingConfig(obj.cluster, obj.locality_picking_policy.map(validateLoadBalancingConfig), obj.endpoint_picking_policy.map(validateLoadBalancingConfig), obj.eds_service_name, obj.lrs_load_reporting_server_name, obj.max_concurrent_requests, validatedOutlierDetectionConfig); } } @@ -449,10 +468,15 @@ export class EdsLoadBalancer implements LoadBalancer { } } + const weightedTargetConfig = new WeightedTargetLoadBalancingConfig(childTargets); + let outlierDetectionConfig: OutlierDetectionLoadBalancingConfig | undefined; + if (EXPERIMENTAL_OUTLIER_DETECTION) { + outlierDetectionConfig = this.lastestConfig.getOutlierDetection()?.copyWithChildPolicy([weightedTargetConfig]); + } + const priorityChildConfig = outlierDetectionConfig ?? weightedTargetConfig; + priorityChildren.set(newPriorityName, { - config: [ - new WeightedTargetLoadBalancingConfig(childTargets), - ], + config: [priorityChildConfig], }); } /* Contract the priority names array if it is sparse. This config only diff --git a/packages/grpc-js-xds/src/xds-stream-state/cds-state.ts b/packages/grpc-js-xds/src/xds-stream-state/cds-state.ts index 7737dc96..8c3c4d73 100644 --- a/packages/grpc-js-xds/src/xds-stream-state/cds-state.ts +++ b/packages/grpc-js-xds/src/xds-stream-state/cds-state.ts @@ -16,8 +16,11 @@ */ import { experimental, logVerbosity, StatusObject } from "@grpc/grpc-js"; +import { EXPERIMENTAL_OUTLIER_DETECTION } from "../environment"; import { Cluster__Output } from "../generated/envoy/config/cluster/v3/Cluster"; import { Any__Output } from "../generated/google/protobuf/Any"; +import { Duration__Output } from "../generated/google/protobuf/Duration"; +import { UInt32Value__Output } from "../generated/google/protobuf/UInt32Value"; import { EdsState } from "./eds-state"; import { HandleResponseResult, RejectedResourceEntry, ResourcePair, Watcher, XdsStreamState } from "./xds-stream-state"; @@ -102,6 +105,26 @@ export class CdsState implements XdsStreamState { return Array.from(this.watchers.keys()); } + private validateNonnegativeDuration(duration: Duration__Output | null): boolean { + if (!duration) { + return true; + } + /* The maximum values here come from the official Protobuf documentation: + * https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Duration + */ + return Number(duration.seconds) >= 0 && + Number(duration.seconds) <= 315_576_000_000 && + duration.nanos >= 0 && + duration.nanos <= 999_999_999; + } + + private validatePercentage(percentage: UInt32Value__Output | null): boolean { + if (!percentage) { + return true; + } + return percentage.value >=0 && percentage.value <= 100; + } + private validateResponse(message: Cluster__Output): boolean { if (message.type !== 'EDS') { return false; @@ -117,6 +140,31 @@ export class CdsState implements XdsStreamState { return false; } } + if (EXPERIMENTAL_OUTLIER_DETECTION) { + if (message.outlier_detection) { + if (!this.validateNonnegativeDuration(message.outlier_detection.interval)) { + return false; + } + if (!this.validateNonnegativeDuration(message.outlier_detection.base_ejection_time)) { + return false; + } + if (!this.validateNonnegativeDuration(message.outlier_detection.max_ejection_time)) { + return false; + } + if (!this.validatePercentage(message.outlier_detection.max_ejection_percent)) { + return false; + } + if (!this.validatePercentage(message.outlier_detection.enforcing_success_rate)) { + return false; + } + if (!this.validatePercentage(message.outlier_detection.failure_percentage_threshold)) { + return false; + } + if (!this.validatePercentage(message.outlier_detection.enforcing_failure_percentage)) { + return false; + } + } + } return true; } diff --git a/packages/grpc-js/src/experimental.ts b/packages/grpc-js/src/experimental.ts index a9d76406..fcafbeb0 100644 --- a/packages/grpc-js/src/experimental.ts +++ b/packages/grpc-js/src/experimental.ts @@ -6,7 +6,7 @@ export { ConfigSelector, } from './resolver'; export { GrpcUri, uriToString } from './uri-parser'; -export { Duration } from './duration'; +export { Duration, durationToMs } from './duration'; export { ServiceConfig } from './service-config'; export { BackoffTimeout } from './backoff-timeout'; export { @@ -35,4 +35,5 @@ export { Call as CallStream } from './call-stream'; export { Filter, BaseFilter, FilterFactory } from './filter'; export { FilterStackFactory } from './filter-stack'; export { registerAdminService } from './admin'; -export { SubchannelInterface, BaseSubchannelWrapper, ConnectivityStateListener } from './subchannel-interface' +export { SubchannelInterface, BaseSubchannelWrapper, ConnectivityStateListener } from './subchannel-interface'; +export { OutlierDetectionLoadBalancingConfig, SuccessRateEjectionConfig, FailurePercentageEjectionConfig } from './load-balancer-outlier-detection'; diff --git a/packages/grpc-js/src/load-balancer-outlier-detection.ts b/packages/grpc-js/src/load-balancer-outlier-detection.ts index ffca1c68..e69e2ef9 100644 --- a/packages/grpc-js/src/load-balancer-outlier-detection.ts +++ b/packages/grpc-js/src/load-balancer-outlier-detection.ts @@ -34,14 +34,14 @@ const TYPE_NAME = 'outlier_detection'; const OUTLIER_DETECTION_ENABLED = process.env.GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION === 'true'; -interface SuccessRateEjectionConfig { +export interface SuccessRateEjectionConfig { readonly stdev_factor: number; readonly enforcement_percentage: number; readonly minimum_hosts: number; readonly request_volume: number; } -interface FailurePercentageEjectionConfig { +export interface FailurePercentageEjectionConfig { readonly threshold: number; readonly enforcement_percentage: number; readonly minimum_hosts: number; @@ -92,15 +92,29 @@ function validatePercentage(obj: any, fieldName: string, objectName?: string) { } export class OutlierDetectionLoadBalancingConfig implements LoadBalancingConfig { + private readonly intervalMs: number; + private readonly baseEjectionTimeMs: number; + private readonly maxEjectionTimeMs: number; + private readonly maxEjectionPercent: number; + private readonly successRateEjection: SuccessRateEjectionConfig | null; + private readonly failurePercentageEjection: FailurePercentageEjectionConfig | null; + constructor( - private readonly intervalMs: number, - private readonly baseEjectionTimeMs: number, - private readonly maxEjectionTimeMs: number, - private readonly maxEjectionPercent: number, - private readonly successRateEjection: SuccessRateEjectionConfig | null, - private readonly failurePercentageEjection: FailurePercentageEjectionConfig | null, + intervalMs: number | null, + baseEjectionTimeMs: number | null, + maxEjectionTimeMs: number | null, + maxEjectionPercent: number | null, + successRateEjection: Partial | null, + failurePercentageEjection: Partial | null, private readonly childPolicy: LoadBalancingConfig[] - ) {} + ) { + this.intervalMs = intervalMs ?? 10_000; + this.baseEjectionTimeMs = baseEjectionTimeMs ?? 30_000; + this.maxEjectionTimeMs = maxEjectionTimeMs ?? 300_000; + this.maxEjectionPercent = maxEjectionPercent ?? 10; + this.successRateEjection = successRateEjection ? {...defaultSuccessRateEjectionConfig, ...successRateEjection} : null; + this.failurePercentageEjection = failurePercentageEjection ? {...defaultFailurePercentageEjectionConfig, ...failurePercentageEjection}: null; + } getLoadBalancerName(): string { return TYPE_NAME; } @@ -137,6 +151,11 @@ export class OutlierDetectionLoadBalancingConfig implements LoadBalancingConfig getChildPolicy(): LoadBalancingConfig[] { return this.childPolicy; } + + copyWithChildPolicy(childPolicy: LoadBalancingConfig[]): OutlierDetectionLoadBalancingConfig { + return new OutlierDetectionLoadBalancingConfig(this.intervalMs, this.baseEjectionTimeMs, this.maxEjectionTimeMs, this.maxEjectionPercent, this.successRateEjection, this.failurePercentageEjection, childPolicy); + } + static createFromJson(obj: any): OutlierDetectionLoadBalancingConfig { validatePositiveDuration(obj, 'interval'); validatePositiveDuration(obj, 'base_ejection_time'); @@ -162,12 +181,12 @@ export class OutlierDetectionLoadBalancingConfig implements LoadBalancingConfig } return new OutlierDetectionLoadBalancingConfig( - obj.interval ? durationToMs(obj.interval) : 10_000, - obj.base_ejection_time ? durationToMs(obj.base_ejection_time) : 30_000, - obj.max_ejection_time ? durationToMs(obj.max_ejection_time) : 300_000, - obj.max_ejection_percent ?? 10, - obj.success_rate_ejection ? {...defaultSuccessRateEjectionConfig, ...obj.success_rate_ejection} : null, - obj.failure_percentage_ejection ? {...defaultFailurePercentageEjectionConfig, ...obj.failure_percentage_ejection} : null, + obj.interval ? durationToMs(obj.interval) : null, + obj.base_ejection_time ? durationToMs(obj.base_ejection_time) : null, + obj.max_ejection_time ? durationToMs(obj.max_ejection_time) : null, + obj.max_ejection_percent ?? null, + obj.success_rate_ejection, + obj.failure_percentage_ejection, obj.child_policy.map(validateLoadBalancingConfig) ); }