grpc
/
grpc-node
mirror of https://github.com/grpc/grpc-node.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2202 from murgatroid99/grpc-js-xds_priority_fixes 

grpc-js-xds: Make various fixes to the priority LB policy
This commit is contained in:
Michael Lumish 2022-08-22 16:06:09 -07:00 committed by GitHub
parent 552b710b93 e1b2cad25e
commit ae53dd1a19
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 56 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
packages/grpc-js-xds/src/load-balancer-priority.ts
View File

@ -115,7 +115,6 @@ interface PriorityChildBalancer {
resetBackoff(): void; resetBackoff(): void;
deactivate(): void; deactivate(): void;
maybeReactivate(): void; maybeReactivate(): void;
cancelFailoverTimer(): void;
isFailoverTimerPending(): boolean; isFailoverTimerPending(): boolean;
getConnectivityState(): ConnectivityState; getConnectivityState(): ConnectivityState;
getPicker(): Picker; getPicker(): Picker;
@ -138,6 +137,7 @@ export class PriorityLoadBalancer implements LoadBalancer {
private childBalancer: ChildLoadBalancerHandler; private childBalancer: ChildLoadBalancerHandler;
private failoverTimer: NodeJS.Timer | null = null; private failoverTimer: NodeJS.Timer | null = null;
private deactivationTimer: NodeJS.Timer | null = null; private deactivationTimer: NodeJS.Timer | null = null;
private seenReadyOrIdleSinceTransientFailure = false;
constructor(private parent: PriorityLoadBalancer, private name: string) { constructor(private parent: PriorityLoadBalancer, private name: string) {
this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, { this.childBalancer = new ChildLoadBalancerHandler(experimental.createChildChannelControlHelper(this.parent.channelControlHelper, {
updateState: (connectivityState: ConnectivityState, picker: Picker) => { updateState: (connectivityState: ConnectivityState, picker: Picker) => {
@ -145,12 +145,24 @@ export class PriorityLoadBalancer implements LoadBalancer {
}, },
})); }));
this.picker = new QueuePicker(this.childBalancer); this.picker = new QueuePicker(this.childBalancer);
this.startFailoverTimer();
} }
private updateState(connectivityState: ConnectivityState, picker: Picker) { private updateState(connectivityState: ConnectivityState, picker: Picker) {
trace('Child ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]); trace('Child ' + this.name + ' ' + ConnectivityState[this.connectivityState] + ' -> ' + ConnectivityState[connectivityState]);
this.connectivityState = connectivityState; this.connectivityState = connectivityState;
this.picker = picker; this.picker = picker;
if (connectivityState === ConnectivityState.CONNECTING) {
if (this.seenReadyOrIdleSinceTransientFailure && this.failoverTimer === null) {
this.startFailoverTimer();
}
} else if (connectivityState === ConnectivityState.READY || connectivityState === ConnectivityState.IDLE) {
this.seenReadyOrIdleSinceTransientFailure = true;
this.cancelFailoverTimer();
} else if (connectivityState === ConnectivityState.TRANSIENT_FAILURE) {
this.seenReadyOrIdleSinceTransientFailure = false;
this.cancelFailoverTimer();
}
this.parent.onChildStateChange(this); this.parent.onChildStateChange(this);
} }
@ -174,13 +186,9 @@ export class PriorityLoadBalancer implements LoadBalancer {
attributes: { [key: string]: unknown } attributes: { [key: string]: unknown }
): void { ): void {
this.childBalancer.updateAddressList(addressList, lbConfig, attributes); this.childBalancer.updateAddressList(addressList, lbConfig, attributes);
this.startFailoverTimer();
} }
exitIdle() { exitIdle() {
if (this.connectivityState === ConnectivityState.IDLE) {
this.startFailoverTimer();
}
this.childBalancer.exitIdle(); this.childBalancer.exitIdle();
} }
@ -204,7 +212,7 @@ export class PriorityLoadBalancer implements LoadBalancer {
} }
} }
cancelFailoverTimer() { private cancelFailoverTimer() {
if (this.failoverTimer !== null) { if (this.failoverTimer !== null) {
clearTimeout(this.failoverTimer); clearTimeout(this.failoverTimer);
this.failoverTimer = null; this.failoverTimer = null;
@ -267,6 +275,8 @@ export class PriorityLoadBalancer implements LoadBalancer {
*/ */
private currentChildFromBeforeUpdate: PriorityChildBalancer | null = null; private currentChildFromBeforeUpdate: PriorityChildBalancer | null = null;
private updatesPaused = false;
constructor(private channelControlHelper: ChannelControlHelper) {} constructor(private channelControlHelper: ChannelControlHelper) {}
private updateState(state: ConnectivityState, picker: Picker) { private updateState(state: ConnectivityState, picker: Picker) {
@ -286,6 +296,9 @@ export class PriorityLoadBalancer implements LoadBalancer {
private onChildStateChange(child: PriorityChildBalancer) { private onChildStateChange(child: PriorityChildBalancer) {
const childState = child.getConnectivityState(); const childState = child.getConnectivityState();
trace('Child ' + child.getName() + ' transitioning to ' + ConnectivityState[childState]); trace('Child ' + child.getName() + ' transitioning to ' + ConnectivityState[childState]);
if (this.updatesPaused) {
return;
}
if (child === this.currentChildFromBeforeUpdate) { if (child === this.currentChildFromBeforeUpdate) {
if ( if (
childState === ConnectivityState.READY || childState === ConnectivityState.READY ||
@ -294,40 +307,11 @@ export class PriorityLoadBalancer implements LoadBalancer {
this.updateState(childState, child.getPicker()); this.updateState(childState, child.getPicker());
} else { } else {
this.currentChildFromBeforeUpdate = null; this.currentChildFromBeforeUpdate = null;
this.tryNextPriority(true); this.choosePriority();
} }
return; return;
} }
const childPriority = this.priorities.indexOf(child.getName()); this.choosePriority();
if (childPriority < 0) {
// child is not in the priority list, ignore updates
return;
}
if (this.currentPriority !== null && childPriority > this.currentPriority) {
// child is lower priority than the currently selected child, ignore updates
return;
}
if (childState === ConnectivityState.TRANSIENT_FAILURE) {
/* Report connecting if and only if the currently selected child is the
* one entering TRANSIENT_FAILURE */
this.tryNextPriority(childPriority === this.currentPriority);
return;
}
if (this.currentPriority === null || childPriority < this.currentPriority) {
/* In this case, either there is no currently selected child or this
* child is higher priority than the currently selected child, so we want
* to switch to it if it is READY or IDLE. */
if (
childState === ConnectivityState.READY ||
childState === ConnectivityState.IDLE
) {
this.selectPriority(childPriority);
}
return;
}
/* The currently selected child has updated state to something other than
* TRANSIENT_FAILURE, so we pass that update along */
this.updateState(childState, child.getPicker());
} }
private deleteChild(child: PriorityChildBalancer) { private deleteChild(child: PriorityChildBalancer) {
@ -335,7 +319,7 @@ export class PriorityLoadBalancer implements LoadBalancer {
this.currentChildFromBeforeUpdate = null; this.currentChildFromBeforeUpdate = null;
/* If we get to this point, the currentChildFromBeforeUpdate was still in /* If we get to this point, the currentChildFromBeforeUpdate was still in
* use, so we are still trying to connect to the specified priorities */ * use, so we are still trying to connect to the specified priorities */
this.tryNextPriority(true); this.choosePriority();
} }
} }
@ -348,7 +332,6 @@ export class PriorityLoadBalancer implements LoadBalancer {
private selectPriority(priority: number) { private selectPriority(priority: number) {
this.currentPriority = priority; this.currentPriority = priority;
const chosenChild = this.children.get(this.priorities[priority])!; const chosenChild = this.children.get(this.priorities[priority])!;
chosenChild.cancelFailoverTimer();
this.updateState( this.updateState(
chosenChild.getConnectivityState(), chosenChild.getConnectivityState(),
chosenChild.getPicker() chosenChild.getPicker()
@ -360,20 +343,22 @@ export class PriorityLoadBalancer implements LoadBalancer {
} }
} }
/** private choosePriority() {
* Check each child in priority order until we find one to use if (this.priorities.length === 0) {
* @param reportConnecting Whether we should report a CONNECTING state if we this.updateState(ConnectivityState.TRANSIENT_FAILURE, new UnavailablePicker({code: Status.UNAVAILABLE, details: 'priority policy has empty priority list', metadata: new Metadata()}));
* stop before picking a specific child. This should be true when we have return;
* not already selected a child. }
*/
private tryNextPriority(reportConnecting: boolean) { this.currentPriority = null;
for (const [index, childName] of this.priorities.entries()) { for (const [priority, childName] of this.priorities.entries()) {
trace('Trying priority ' + priority + ' child ' + childName);
let child = this.children.get(childName); let child = this.children.get(childName);
/* If the child doesn't already exist, create it and update it. */ /* If the child doesn't already exist, create it and update it. */
if (child === undefined) { if (child === undefined) {
if (reportConnecting) { if (this.currentChildFromBeforeUpdate === null) {
this.updateState(ConnectivityState.CONNECTING, new QueuePicker(this)); this.updateState(ConnectivityState.CONNECTING, new QueuePicker(this));
} }
this.currentPriority = priority;
child = new this.PriorityChildImpl(this, childName); child = new this.PriorityChildImpl(this, childName);
this.children.set(childName, child); this.children.set(childName, child);
const childUpdate = this.latestUpdates.get(childName); const childUpdate = this.latestUpdates.get(childName);
@ -383,6 +368,7 @@ export class PriorityLoadBalancer implements LoadBalancer {
childUpdate.lbConfig, childUpdate.lbConfig,
this.latestAttributes this.latestAttributes
); );
return;
} }
} }
/* We're going to try to use this child, so reactivate it if it has been /* We're going to try to use this child, so reactivate it if it has been
@ -392,28 +378,33 @@ export class PriorityLoadBalancer implements LoadBalancer {
child.getConnectivityState() === ConnectivityState.READY || child.getConnectivityState() === ConnectivityState.READY ||
child.getConnectivityState() === ConnectivityState.IDLE child.getConnectivityState() === ConnectivityState.IDLE
) { ) {
this.selectPriority(index); this.selectPriority(priority);
return; return;
} }
if (child.isFailoverTimerPending()) { if (child.isFailoverTimerPending()) {
this.currentPriority = priority;
if (this.currentChildFromBeforeUpdate === null) {
/* This child is still trying to connect. Wait until its failover timer /* This child is still trying to connect. Wait until its failover timer
* has ended to continue to the next one */ * has ended to continue to the next one */
if (reportConnecting) {
this.updateState(ConnectivityState.CONNECTING, new QueuePicker(this)); this.updateState(ConnectivityState.CONNECTING, new QueuePicker(this));
}
return; return;
} }
} }
this.currentPriority = null; }
this.currentChildFromBeforeUpdate = null;
this.updateState( /* If we didn't find any priority to try, pick the first one in the state
ConnectivityState.TRANSIENT_FAILURE, * CONNECTING */
new UnavailablePicker({ for (const [priority, childName] of this.priorities.entries()) {
code: Status.UNAVAILABLE, let child = this.children.get(childName)!;
details: 'No ready priority', if (child.getConnectivityState() === ConnectivityState.CONNECTING) {
metadata: new Metadata(), this.updateState(child.getConnectivityState(), child.getPicker());
}) return;
); }
}
// Did not find any child in CONNECTING, delegate to last child
const child = this.children.get(this.priorities[this.priorities.length - 1])!;
this.updateState(child.getConnectivityState(), child.getPicker());
} }
updateAddressList( updateAddressList(
@ -464,6 +455,7 @@ export class PriorityLoadBalancer implements LoadBalancer {
this.latestAttributes = attributes; this.latestAttributes = attributes;
this.latestUpdates.clear(); this.latestUpdates.clear();
this.priorities = lbConfig.getPriorities(); this.priorities = lbConfig.getPriorities();
this.updatesPaused = true;
/* Pair up the new child configs with the corresponding address lists, and /* Pair up the new child configs with the corresponding address lists, and
* update all existing children with their new configs */ * update all existing children with their new configs */
for (const [childName, childConfig] of lbConfig.getChildren()) { for (const [childName, childConfig] of lbConfig.getChildren()) {
@ -492,8 +484,8 @@ export class PriorityLoadBalancer implements LoadBalancer {
child.deactivate(); child.deactivate();
} }
} }
// Only report connecting if there are no existing children this.updatesPaused = false;
this.tryNextPriority(this.children.size === 0); this.choosePriority();
} }
exitIdle(): void { exitIdle(): void {
if (this.currentPriority !== null) { if (this.currentPriority !== null) {