grpc
/
grpc-node
mirror of https://github.com/grpc/grpc-node.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

grpc-js-xds: Add outlier detection configuration handling

This commit is contained in:
Michael Lumish 2022-03-18 12:40:30 -07:00
parent 0e332863f5
commit b5b0703bcd
6 changed files with 177 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/grpc-js-xds/src/environment.ts
View File

@ -15,4 +15,5 @@
*
*/
export const EXPERIMENTAL_FAULT_INJECTION = (process.env.GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION ?? 'true') === 'true';
export const EXPERIMENTAL_FAULT_INJECTION = (process.env.GRPC_XDS_EXPERIMENTAL_FAULT_INJECTION ?? 'true') === 'true';
export const EXPERIMENTAL_OUTLIER_DETECTION = process.env.GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION === 'true';

62
packages/grpc-js-xds/src/load-balancer-cds.ts
View File

@ -25,8 +25,14 @@ import LoadBalancer = experimental.LoadBalancer;
import ChannelControlHelper = experimental.ChannelControlHelper;
import registerLoadBalancerType = experimental.registerLoadBalancerType;
import LoadBalancingConfig = experimental.LoadBalancingConfig;
import OutlierDetectionLoadBalancingConfig = experimental.OutlierDetectionLoadBalancingConfig;
import SuccessRateEjectionConfig = experimental.SuccessRateEjectionConfig;
import FailurePercentageEjectionConfig = experimental.FailurePercentageEjectionConfig;
import { EdsLoadBalancingConfig } from './load-balancer-eds';
import { Watcher } from './xds-stream-state/xds-stream-state';
import { OutlierDetection__Output } from './generated/envoy/config/cluster/v3/OutlierDetection';
import { Duration__Output } from './generated/google/protobuf/Duration';
import { EXPERIMENTAL_OUTLIER_DETECTION } from './environment';
const TRACER_NAME = 'cds_balancer';
@ -64,6 +70,52 @@ export class CdsLoadBalancingConfig implements LoadBalancingConfig {
}
}
function durationToMs(duration: Duration__Output): number {
return (Number(duration.seconds) * 1_000 + duration.nanos / 1_000_000) | 0;
}
function translateOutlierDetectionConfig(outlierDetection: OutlierDetection__Output | null): OutlierDetectionLoadBalancingConfig | undefined {
if (!EXPERIMENTAL_OUTLIER_DETECTION) {
return undefined;
}
if (!outlierDetection) {
/* No-op outlier detection config, with max possible interval and no
* ejection criteria configured. */
return new OutlierDetectionLoadBalancingConfig(~(1<<31), null, null, null, null, null, []);
}
let successRateConfig: Partial<SuccessRateEjectionConfig> | null = null;
/* Success rate ejection is enabled by default, so we only disable it if
* enforcing_success_rate is set and it has the value 0 */
if (!outlierDetection.enforcing_success_rate || outlierDetection.enforcing_success_rate.value > 0) {
successRateConfig = {
enforcement_percentage: outlierDetection.enforcing_success_rate?.value,
minimum_hosts: outlierDetection.success_rate_minimum_hosts?.value,
request_volume: outlierDetection.success_rate_request_volume?.value,
stdev_factor: outlierDetection.success_rate_stdev_factor?.value
};
}
let failurePercentageConfig: Partial<FailurePercentageEjectionConfig> | null = null;
/* Failure percentage ejection is disabled by default, so we only enable it
* if enforcing_failure_percentage is set and it has a value greater than 0 */
if (outlierDetection.enforcing_failure_percentage && outlierDetection.enforcing_failure_percentage.value > 0) {
failurePercentageConfig = {
enforcement_percentage: outlierDetection.enforcing_failure_percentage.value,
minimum_hosts: outlierDetection.failure_percentage_minimum_hosts?.value,
request_volume: outlierDetection.failure_percentage_request_volume?.value,
threshold: outlierDetection.failure_percentage_threshold?.value
}
}
return new OutlierDetectionLoadBalancingConfig(
outlierDetection.interval ? durationToMs(outlierDetection.interval) : null,
outlierDetection.base_ejection_time ? durationToMs(outlierDetection.base_ejection_time) : null,
outlierDetection.max_ejection_time ? durationToMs(outlierDetection.max_ejection_time) : null,
outlierDetection.max_ejection_percent?.value ?? null,
successRateConfig,
failurePercentageConfig,
[]
);
}
export class CdsLoadBalancer implements LoadBalancer {
private childBalancer: ChildLoadBalancerHandler;
private watcher: Watcher<Cluster__Output>;
@ -90,7 +142,15 @@ export class CdsLoadBalancer implements LoadBalancer {
* used for load reporting as for other xDS operations. Setting
* lrsLoadReportingServerName to the empty string sets that behavior.
* Otherwise, if the field is omitted, load reporting is disabled. */
const edsConfig: EdsLoadBalancingConfig = new EdsLoadBalancingConfig(update.name, [], [], update.eds_cluster_config!.service_name === '' ? undefined : update.eds_cluster_config!.service_name, update.lrs_server?.self ? '' : undefined, maxConcurrentRequests);
const edsConfig: EdsLoadBalancingConfig = new EdsLoadBalancingConfig(
/* cluster= */ update.name,
/* localityPickingPolicy= */ [],
/* endpointPickingPolicy= */ [],
/* edsServiceName= */ update.eds_cluster_config!.service_name === '' ? undefined : update.eds_cluster_config!.service_name,
/* lrsLoadReportingServerName= */update.lrs_server?.self ? '' : undefined,
/* maxConcurrentRequests= */ maxConcurrentRequests,
/* outlierDetection= */ translateOutlierDetectionConfig(update.outlier_detection)
);
trace('Child update EDS config: ' + JSON.stringify(edsConfig));
this.childBalancer.updateAddressList(
[],

34
packages/grpc-js-xds/src/load-balancer-eds.ts
View File

@ -38,6 +38,8 @@ import Filter = experimental.Filter;
import BaseFilter = experimental.BaseFilter;
import FilterFactory = experimental.FilterFactory;
import CallStream = experimental.CallStream;
import OutlierDetectionLoadBalancingConfig = experimental.OutlierDetectionLoadBalancingConfig;
import { EXPERIMENTAL_OUTLIER_DETECTION } from './environment';
const TRACER_NAME = 'eds_balancer';
@ -71,12 +73,15 @@ export class EdsLoadBalancingConfig implements LoadBalancingConfig {
if (this.lrsLoadReportingServerName !== undefined) {
jsonObj.lrs_load_reporting_server_name = this.lrsLoadReportingServerName;
}
if (this.outlierDetection !== undefined) {
jsonObj.outlier_detection = this.outlierDetection.toJsonObject();
}
return {
[TYPE_NAME]: jsonObj
};
}
constructor(private cluster: string, private localityPickingPolicy: LoadBalancingConfig[], private endpointPickingPolicy: LoadBalancingConfig[], private edsServiceName?: string, private lrsLoadReportingServerName?: string, maxConcurrentRequests?: number) {
constructor(private cluster: string, private localityPickingPolicy: LoadBalancingConfig[], private endpointPickingPolicy: LoadBalancingConfig[], private edsServiceName?: string, private lrsLoadReportingServerName?: string, maxConcurrentRequests?: number, private outlierDetection?: OutlierDetectionLoadBalancingConfig) {
this.maxConcurrentRequests = maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS;
}
@ -104,6 +109,10 @@ export class EdsLoadBalancingConfig implements LoadBalancingConfig {
return this.maxConcurrentRequests;
}
getOutlierDetection() {
return this.outlierDetection;
}
static createFromJson(obj: any): EdsLoadBalancingConfig {
if (!('cluster' in obj && typeof obj.cluster === 'string')) {
throw new Error('eds config must have a string field cluster');
@ -123,7 +132,17 @@ export class EdsLoadBalancingConfig implements LoadBalancingConfig {
if ('max_concurrent_requests' in obj && (!obj.max_concurrent_requests === undefined || typeof obj.max_concurrent_requests === 'number')) {
throw new Error('eds config max_concurrent_requests must be a number if provided');
}
return new EdsLoadBalancingConfig(obj.cluster, obj.locality_picking_policy.map(validateLoadBalancingConfig), obj.endpoint_picking_policy.map(validateLoadBalancingConfig), obj.eds_service_name, obj.lrs_load_reporting_server_name, obj.max_concurrent_requests);
let validatedOutlierDetectionConfig: OutlierDetectionLoadBalancingConfig | undefined = undefined;
if (EXPERIMENTAL_OUTLIER_DETECTION) {
if ('outlier_detection' in obj) {
const outlierDetectionConfig = validateLoadBalancingConfig(obj.outlier_detection);
if (!(outlierDetectionConfig instanceof OutlierDetectionLoadBalancingConfig)) {
throw new Error('eds config outlier_detection must be a valid outlier detection config if provided');
}
validatedOutlierDetectionConfig = outlierDetectionConfig;
}
}
return new EdsLoadBalancingConfig(obj.cluster, obj.locality_picking_policy.map(validateLoadBalancingConfig), obj.endpoint_picking_policy.map(validateLoadBalancingConfig), obj.eds_service_name, obj.lrs_load_reporting_server_name, obj.max_concurrent_requests, validatedOutlierDetectionConfig);
}
}
@ -449,10 +468,15 @@ export class EdsLoadBalancer implements LoadBalancer {
}
}
const weightedTargetConfig = new WeightedTargetLoadBalancingConfig(childTargets);
let outlierDetectionConfig: OutlierDetectionLoadBalancingConfig | undefined;
if (EXPERIMENTAL_OUTLIER_DETECTION) {
outlierDetectionConfig = this.lastestConfig.getOutlierDetection()?.copyWithChildPolicy([weightedTargetConfig]);
}
const priorityChildConfig = outlierDetectionConfig ?? weightedTargetConfig;
priorityChildren.set(newPriorityName, {
config: [
new WeightedTargetLoadBalancingConfig(childTargets),
],
config: [priorityChildConfig],
});
}
/* Contract the priority names array if it is sparse. This config only

48
packages/grpc-js-xds/src/xds-stream-state/cds-state.ts
View File

@ -16,8 +16,11 @@
*/
import { experimental, logVerbosity, StatusObject } from "@grpc/grpc-js";
import { EXPERIMENTAL_OUTLIER_DETECTION } from "../environment";
import { Cluster__Output } from "../generated/envoy/config/cluster/v3/Cluster";
import { Any__Output } from "../generated/google/protobuf/Any";
import { Duration__Output } from "../generated/google/protobuf/Duration";
import { UInt32Value__Output } from "../generated/google/protobuf/UInt32Value";
import { EdsState } from "./eds-state";
import { HandleResponseResult, RejectedResourceEntry, ResourcePair, Watcher, XdsStreamState } from "./xds-stream-state";
@ -102,6 +105,26 @@ export class CdsState implements XdsStreamState<Cluster__Output> {
return Array.from(this.watchers.keys());
}
private validateNonnegativeDuration(duration: Duration__Output | null): boolean {
if (!duration) {
return true;
}
/* The maximum values here come from the official Protobuf documentation:
* https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Duration
*/
return Number(duration.seconds) >= 0 &&
Number(duration.seconds) <= 315_576_000_000 &&
duration.nanos >= 0 &&
duration.nanos <= 999_999_999;
}
private validatePercentage(percentage: UInt32Value__Output | null): boolean {
if (!percentage) {
return true;
}
return percentage.value >=0 && percentage.value <= 100;
}
private validateResponse(message: Cluster__Output): boolean {
if (message.type !== 'EDS') {
return false;
@ -117,6 +140,31 @@ export class CdsState implements XdsStreamState<Cluster__Output> {
return false;
}
}
if (EXPERIMENTAL_OUTLIER_DETECTION) {
if (message.outlier_detection) {
if (!this.validateNonnegativeDuration(message.outlier_detection.interval)) {
return false;
}
if (!this.validateNonnegativeDuration(message.outlier_detection.base_ejection_time)) {
return false;
}
if (!this.validateNonnegativeDuration(message.outlier_detection.max_ejection_time)) {
return false;
}
if (!this.validatePercentage(message.outlier_detection.max_ejection_percent)) {
return false;
}
if (!this.validatePercentage(message.outlier_detection.enforcing_success_rate)) {
return false;
}
if (!this.validatePercentage(message.outlier_detection.failure_percentage_threshold)) {
return false;
}
if (!this.validatePercentage(message.outlier_detection.enforcing_failure_percentage)) {
return false;
}
}
}
return true;
}

5
packages/grpc-js/src/experimental.ts
View File

@ -6,7 +6,7 @@ export {
ConfigSelector,
} from './resolver';
export { GrpcUri, uriToString } from './uri-parser';
export { Duration } from './duration';
export { Duration, durationToMs } from './duration';
export { ServiceConfig } from './service-config';
export { BackoffTimeout } from './backoff-timeout';
export {
@ -35,4 +35,5 @@ export { Call as CallStream } from './call-stream';
export { Filter, BaseFilter, FilterFactory } from './filter';
export { FilterStackFactory } from './filter-stack';
export { registerAdminService } from './admin';
export { SubchannelInterface, BaseSubchannelWrapper, ConnectivityStateListener } from './subchannel-interface'
export { SubchannelInterface, BaseSubchannelWrapper, ConnectivityStateListener } from './subchannel-interface';
export { OutlierDetectionLoadBalancingConfig, SuccessRateEjectionConfig, FailurePercentageEjectionConfig } from './load-balancer-outlier-detection';

49
packages/grpc-js/src/load-balancer-outlier-detection.ts
View File

@ -34,14 +34,14 @@ const TYPE_NAME = 'outlier_detection';
const OUTLIER_DETECTION_ENABLED = process.env.GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION === 'true';
interface SuccessRateEjectionConfig {
export interface SuccessRateEjectionConfig {
readonly stdev_factor: number;
readonly enforcement_percentage: number;
readonly minimum_hosts: number;
readonly request_volume: number;
}
interface FailurePercentageEjectionConfig {
export interface FailurePercentageEjectionConfig {
readonly threshold: number;
readonly enforcement_percentage: number;
readonly minimum_hosts: number;
@ -92,15 +92,29 @@ function validatePercentage(obj: any, fieldName: string, objectName?: string) {
}
export class OutlierDetectionLoadBalancingConfig implements LoadBalancingConfig {
private readonly intervalMs: number;
private readonly baseEjectionTimeMs: number;
private readonly maxEjectionTimeMs: number;
private readonly maxEjectionPercent: number;
private readonly successRateEjection: SuccessRateEjectionConfig | null;
private readonly failurePercentageEjection: FailurePercentageEjectionConfig | null;
constructor(
private readonly intervalMs: number,
private readonly baseEjectionTimeMs: number,
private readonly maxEjectionTimeMs: number,
private readonly maxEjectionPercent: number,
private readonly successRateEjection: SuccessRateEjectionConfig | null,
private readonly failurePercentageEjection: FailurePercentageEjectionConfig | null,
intervalMs: number | null,
baseEjectionTimeMs: number | null,
maxEjectionTimeMs: number | null,
maxEjectionPercent: number | null,
successRateEjection: Partial<SuccessRateEjectionConfig> | null,
failurePercentageEjection: Partial<FailurePercentageEjectionConfig> | null,
private readonly childPolicy: LoadBalancingConfig[]
) {}
) {
this.intervalMs = intervalMs ?? 10_000;
this.baseEjectionTimeMs = baseEjectionTimeMs ?? 30_000;
this.maxEjectionTimeMs = maxEjectionTimeMs ?? 300_000;
this.maxEjectionPercent = maxEjectionPercent ?? 10;
this.successRateEjection = successRateEjection ? {...defaultSuccessRateEjectionConfig, ...successRateEjection} : null;
this.failurePercentageEjection = failurePercentageEjection ? {...defaultFailurePercentageEjectionConfig, ...failurePercentageEjection}: null;
}
getLoadBalancerName(): string {
return TYPE_NAME;
}
@ -137,6 +151,11 @@ export class OutlierDetectionLoadBalancingConfig implements LoadBalancingConfig
getChildPolicy(): LoadBalancingConfig[] {
return this.childPolicy;
}
copyWithChildPolicy(childPolicy: LoadBalancingConfig[]): OutlierDetectionLoadBalancingConfig {
return new OutlierDetectionLoadBalancingConfig(this.intervalMs, this.baseEjectionTimeMs, this.maxEjectionTimeMs, this.maxEjectionPercent, this.successRateEjection, this.failurePercentageEjection, childPolicy);
}
static createFromJson(obj: any): OutlierDetectionLoadBalancingConfig {
validatePositiveDuration(obj, 'interval');
validatePositiveDuration(obj, 'base_ejection_time');
@ -162,12 +181,12 @@ export class OutlierDetectionLoadBalancingConfig implements LoadBalancingConfig
}
return new OutlierDetectionLoadBalancingConfig(
obj.interval ? durationToMs(obj.interval) : 10_000,
obj.base_ejection_time ? durationToMs(obj.base_ejection_time) : 30_000,
obj.max_ejection_time ? durationToMs(obj.max_ejection_time) : 300_000,
obj.max_ejection_percent ?? 10,
obj.success_rate_ejection ? {...defaultSuccessRateEjectionConfig, ...obj.success_rate_ejection} : null,
obj.failure_percentage_ejection ? {...defaultFailurePercentageEjectionConfig, ...obj.failure_percentage_ejection} : null,
obj.interval ? durationToMs(obj.interval) : null,
obj.base_ejection_time ? durationToMs(obj.base_ejection_time) : null,
obj.max_ejection_time ? durationToMs(obj.max_ejection_time) : null,
obj.max_ejection_percent ?? null,
obj.success_rate_ejection,
obj.failure_percentage_ejection,
obj.child_policy.map(validateLoadBalancingConfig)
);
}