grpc
/
grpc-go
mirror of https://github.com/grpc/grpc-go.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

xds/internal/balancer/outlierdetection: Switch Outlier Detection to use new duration field (#6286)

This commit is contained in:
Zach Reyes 2023-05-18 14:28:53 -04:00 committed by GitHub
parent 417d4b6895
commit 098b2d00c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 85 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
xds/internal/balancer/cdsbalancer/cdsbalancer.go
View File

@ -308,9 +308,9 @@ func outlierDetectionToConfig(od *xdsresource.OutlierDetection) outlierdetection
}
return outlierdetection.LBConfig{
Interval: od.Interval,
BaseEjectionTime: od.BaseEjectionTime,
MaxEjectionTime: od.MaxEjectionTime,
Interval: internalserviceconfig.Duration(od.Interval),
BaseEjectionTime: internalserviceconfig.Duration(od.BaseEjectionTime),
MaxEjectionTime: internalserviceconfig.Duration(od.MaxEjectionTime),
MaxEjectionPercent: od.MaxEjectionPercent,
SuccessRateEjection: sre,
FailurePercentageEjection: fpe,

24
xds/internal/balancer/cdsbalancer/cdsbalancer_test.go
View File

@ -444,9 +444,9 @@ func (s) TestHandleClusterUpdate(t *testing.T) {
LBPolicy: wrrLocalityLBConfigJSON,
},
wantCCS: edsCCS(serviceName, nil, false, wrrLocalityLBConfig, outlierdetection.LBConfig{
Interval: 10 * time.Second,
BaseEjectionTime: 30 * time.Second,
MaxEjectionTime: 300 * time.Second,
Interval: internalserviceconfig.Duration(10 * time.Second),
BaseEjectionTime: internalserviceconfig.Duration(30 * time.Second),
MaxEjectionTime: internalserviceconfig.Duration(300 * time.Second),
MaxEjectionPercent: 10,
SuccessRateEjection: &outlierdetection.SuccessRateEjection{
StdevFactor: 1900,
@ -918,9 +918,9 @@ func (s) TestOutlierDetectionToConfig(t *testing.T) {
FailurePercentageRequestVolume: 50,
},
odLBCfgWant: outlierdetection.LBConfig{
Interval: 10 * time.Second,
BaseEjectionTime: 30 * time.Second,
MaxEjectionTime: 300 * time.Second,
Interval: internalserviceconfig.Duration(10 * time.Second),
BaseEjectionTime: internalserviceconfig.Duration(30 * time.Second),
MaxEjectionTime: internalserviceconfig.Duration(300 * time.Second),
MaxEjectionPercent: 10,
SuccessRateEjection: nil,
FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
@ -951,9 +951,9 @@ func (s) TestOutlierDetectionToConfig(t *testing.T) {
FailurePercentageRequestVolume: 50,
},
odLBCfgWant: outlierdetection.LBConfig{
Interval: 10 * time.Second,
BaseEjectionTime: 30 * time.Second,
MaxEjectionTime: 300 * time.Second,
Interval: internalserviceconfig.Duration(10 * time.Second),
BaseEjectionTime: internalserviceconfig.Duration(30 * time.Second),
MaxEjectionTime: internalserviceconfig.Duration(300 * time.Second),
MaxEjectionPercent: 10,
SuccessRateEjection: &outlierdetection.SuccessRateEjection{
StdevFactor: 1900,
@ -981,9 +981,9 @@ func (s) TestOutlierDetectionToConfig(t *testing.T) {
FailurePercentageRequestVolume: 50,
},
odLBCfgWant: outlierdetection.LBConfig{
Interval: 10 * time.Second,
BaseEjectionTime: 30 * time.Second,
MaxEjectionTime: 300 * time.Second,
Interval: internalserviceconfig.Duration(10 * time.Second),
BaseEjectionTime: internalserviceconfig.Duration(30 * time.Second),
MaxEjectionTime: internalserviceconfig.Duration(300 * time.Second),
MaxEjectionPercent: 10,
SuccessRateEjection: &outlierdetection.SuccessRateEjection{
StdevFactor: 1900,

18
xds/internal/balancer/outlierdetection/balancer.go
View File

@ -225,9 +225,9 @@ func (b *outlierDetectionBalancer) onIntervalConfig() {
for _, addrInfo := range b.addrs {
addrInfo.callCounter.clear()
}
interval = b.cfg.Interval
interval = time.Duration(b.cfg.Interval)
} else {
interval = b.cfg.Interval - now().Sub(b.timerStartTime)
interval = time.Duration(b.cfg.Interval) - now().Sub(b.timerStartTime)
if interval < 0 {
interval = 0
}
@ -589,14 +589,14 @@ func (b *outlierDetectionBalancer) Target() string {
return b.cc.Target()
}
func max(x, y int64) int64 {
func max(x, y time.Duration) time.Duration {
if x < y {
return y
}
return x
}
func min(x, y int64) int64 {
func min(x, y time.Duration) time.Duration {
if x < y {
return x
}
@ -754,10 +754,10 @@ func (b *outlierDetectionBalancer) intervalTimerAlgorithm() {
// to uneject the address below.
continue
}
et := b.cfg.BaseEjectionTime.Nanoseconds() * addrInfo.ejectionTimeMultiplier
met := max(b.cfg.BaseEjectionTime.Nanoseconds(), b.cfg.MaxEjectionTime.Nanoseconds())
curTimeAfterEt := now().After(addrInfo.latestEjectionTimestamp.Add(time.Duration(min(et, met))))
if curTimeAfterEt {
et := time.Duration(b.cfg.BaseEjectionTime) * time.Duration(addrInfo.ejectionTimeMultiplier)
met := max(time.Duration(b.cfg.BaseEjectionTime), time.Duration(b.cfg.MaxEjectionTime))
uet := addrInfo.latestEjectionTimestamp.Add(min(et, met))
if now().After(uet) {
b.unejectAddress(addrInfo)
}
}
@ -767,7 +767,7 @@ func (b *outlierDetectionBalancer) intervalTimerAlgorithm() {
if b.intervalTimer != nil {
b.intervalTimer.Stop()
}
b.intervalTimer = afterFunc(b.cfg.Interval, b.intervalTimerAlgorithm)
b.intervalTimer = afterFunc(time.Duration(b.cfg.Interval), b.intervalTimerAlgorithm)
}
// addrsWithAtLeastRequestVolume returns a slice of address information of all

93
xds/internal/balancer/outlierdetection/balancer_test.go
View File

@ -37,7 +37,7 @@ import (
"google.golang.org/grpc/internal/channelz"
"google.golang.org/grpc/internal/grpcsync"
"google.golang.org/grpc/internal/grpctest"
internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/resolver"
"google.golang.org/grpc/serviceconfig"
@ -78,7 +78,6 @@ func (s) TestParseConfig(t *testing.T) {
{
name: "noop-lb-config",
input: `{
"interval": 9223372036854775807,
"childPolicy": [
{
"xds_cluster_impl_experimental": {
@ -88,8 +87,7 @@ func (s) TestParseConfig(t *testing.T) {
]
}`,
wantCfg: &LBConfig{
Interval: math.MaxInt64,
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "xds_cluster_impl_experimental",
Config: &clusterimpl.LBConfig{
Cluster: "test_cluster",
@ -100,9 +98,9 @@ func (s) TestParseConfig(t *testing.T) {
{
name: "good-lb-config",
input: `{
"interval": 10000000000,
"baseEjectionTime": 30000000000,
"maxEjectionTime": 300000000000,
"interval": "10s",
"baseEjectionTime": "30s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 10,
"successRateEjection": {
"stdevFactor": 1900,
@ -125,9 +123,9 @@ func (s) TestParseConfig(t *testing.T) {
]
}`,
wantCfg: &LBConfig{
Interval: 10 * time.Second,
BaseEjectionTime: 30 * time.Second,
MaxEjectionTime: 300 * time.Second,
Interval: iserviceconfig.Duration(10 * time.Second),
BaseEjectionTime: iserviceconfig.Duration(30 * time.Second),
MaxEjectionTime: iserviceconfig.Duration(300 * time.Second),
MaxEjectionPercent: 10,
SuccessRateEjection: &SuccessRateEjection{
StdevFactor: 1900,
@ -141,7 +139,7 @@ func (s) TestParseConfig(t *testing.T) {
MinimumHosts: 5,
RequestVolume: 50,
},
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "xds_cluster_impl_experimental",
Config: &clusterimpl.LBConfig{
Cluster: "test_cluster",
@ -151,18 +149,18 @@ func (s) TestParseConfig(t *testing.T) {
},
{
name: "interval-is-negative",
input: `{"interval": -10}`,
wantErr: "OutlierDetectionLoadBalancingConfig.interval = -10ns; must be >= 0",
input: `{"interval": "-10s"}`,
wantErr: "OutlierDetectionLoadBalancingConfig.interval = -10s; must be >= 0",
},
{
name: "base-ejection-time-is-negative",
input: `{"baseEjectionTime": -10}`,
wantErr: "OutlierDetectionLoadBalancingConfig.base_ejection_time = -10ns; must be >= 0",
input: `{"baseEjectionTime": "-10s"}`,
wantErr: "OutlierDetectionLoadBalancingConfig.base_ejection_time = -10s; must be >= 0",
},
{
name: "max-ejection-time-is-negative",
input: `{"maxEjectionTime": -10}`,
wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_time = -10ns; must be >= 0",
input: `{"maxEjectionTime": "-10s"}`,
wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_time = -10s; must be >= 0",
},
{
name: "max-ejection-percent-is-greater-than-100",
@ -199,9 +197,9 @@ func (s) TestParseConfig(t *testing.T) {
{
name: "child-policy-not-present",
input: `{
"interval": 10000000000,
"baseEjectionTime": 30000000000,
"maxEjectionTime": 300000000000,
"interval": "10s",
"baseEjectionTime": "30s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 10,
"successRateEjection": {
"stdevFactor": 1900,
@ -221,7 +219,6 @@ func (s) TestParseConfig(t *testing.T) {
{
name: "child-policy-present-but-parse-error",
input: `{
"interval": 9223372036854775807,
"childPolicy": [
{
"errParseConfigBalancer": {
@ -235,7 +232,6 @@ func (s) TestParseConfig(t *testing.T) {
{
name: "no-supported-child-policy",
input: `{
"interval": 9223372036854775807,
"childPolicy": [
{
"doesNotExistBalancer": {
@ -258,7 +254,7 @@ func (s) TestParseConfig(t *testing.T) {
]
}`,
wantCfg: &LBConfig{
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: "xds_cluster_impl_experimental",
Config: &clusterimpl.LBConfig{
Cluster: "test_cluster",
@ -362,8 +358,7 @@ func (s) TestChildBasicOperations(t *testing.T) {
// it's first update.
od.UpdateClientConnState(balancer.ClientConnState{
BalancerConfig: &LBConfig{
Interval: math.MaxInt64,
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name() + "child1",
Config: bc,
},
@ -386,7 +381,7 @@ func (s) TestChildBasicOperations(t *testing.T) {
od.UpdateClientConnState(balancer.ClientConnState{
BalancerConfig: &LBConfig{
Interval: math.MaxInt64,
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name() + "child2",
Config: emptyChildConfig{},
},
@ -475,9 +470,9 @@ func (s) TestUpdateAddresses(t *testing.T) {
},
},
BalancerConfig: &LBConfig{
Interval: 10 * time.Second,
BaseEjectionTime: 30 * time.Second,
MaxEjectionTime: 300 * time.Second,
Interval: iserviceconfig.Duration(10 * time.Second),
BaseEjectionTime: iserviceconfig.Duration(30 * time.Second),
MaxEjectionTime: iserviceconfig.Duration(300 * time.Second),
MaxEjectionPercent: 10,
FailurePercentageEjection: &FailurePercentageEjection{
Threshold: 50,
@ -485,7 +480,7 @@ func (s) TestUpdateAddresses(t *testing.T) {
MinimumHosts: 2,
RequestVolume: 3,
},
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name(),
Config: emptyChildConfig{},
},
@ -651,14 +646,14 @@ func (s) TestDurationOfInterval(t *testing.T) {
od.UpdateClientConnState(balancer.ClientConnState{
BalancerConfig: &LBConfig{
Interval: 8 * time.Second,
Interval: iserviceconfig.Duration(8 * time.Second),
SuccessRateEjection: &SuccessRateEjection{
StdevFactor: 1900,
EnforcementPercentage: 100,
MinimumHosts: 5,
RequestVolume: 100,
},
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name(),
Config: emptyChildConfig{},
},
@ -691,14 +686,14 @@ func (s) TestDurationOfInterval(t *testing.T) {
// interval timer of ~4 seconds.
od.UpdateClientConnState(balancer.ClientConnState{
BalancerConfig: &LBConfig{
Interval: 9 * time.Second,
Interval: iserviceconfig.Duration(9 * time.Second),
SuccessRateEjection: &SuccessRateEjection{
StdevFactor: 1900,
EnforcementPercentage: 100,
MinimumHosts: 5,
RequestVolume: 100,
},
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name(),
Config: emptyChildConfig{},
},
@ -718,8 +713,8 @@ func (s) TestDurationOfInterval(t *testing.T) {
// interval timer at all due to it being a no-op.
od.UpdateClientConnState(balancer.ClientConnState{
BalancerConfig: &LBConfig{
Interval: 10 * time.Second,
ChildPolicy: &internalserviceconfig.BalancerConfig{
Interval: iserviceconfig.Duration(10 * time.Second),
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name(),
Config: emptyChildConfig{},
},
@ -793,8 +788,8 @@ func (s) TestEjectUnejectSuccessRate(t *testing.T) {
},
BalancerConfig: &LBConfig{
Interval: math.MaxInt64, // so the interval will never run unless called manually in test.
BaseEjectionTime: 30 * time.Second,
MaxEjectionTime: 300 * time.Second,
BaseEjectionTime: iserviceconfig.Duration(30 * time.Second),
MaxEjectionTime: iserviceconfig.Duration(300 * time.Second),
MaxEjectionPercent: 10,
FailurePercentageEjection: &FailurePercentageEjection{
Threshold: 50,
@ -802,7 +797,7 @@ func (s) TestEjectUnejectSuccessRate(t *testing.T) {
MinimumHosts: 3,
RequestVolume: 3,
},
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name(),
Config: emptyChildConfig{},
},
@ -997,8 +992,8 @@ func (s) TestEjectFailureRate(t *testing.T) {
},
BalancerConfig: &LBConfig{
Interval: math.MaxInt64, // so the interval will never run unless called manually in test.
BaseEjectionTime: 30 * time.Second,
MaxEjectionTime: 300 * time.Second,
BaseEjectionTime: iserviceconfig.Duration(30 * time.Second),
MaxEjectionTime: iserviceconfig.Duration(300 * time.Second),
MaxEjectionPercent: 10,
SuccessRateEjection: &SuccessRateEjection{
StdevFactor: 500,
@ -1006,7 +1001,7 @@ func (s) TestEjectFailureRate(t *testing.T) {
MinimumHosts: 3,
RequestVolume: 3,
},
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name(),
Config: emptyChildConfig{},
},
@ -1103,10 +1098,10 @@ func (s) TestEjectFailureRate(t *testing.T) {
},
BalancerConfig: &LBConfig{
Interval: math.MaxInt64,
BaseEjectionTime: 30 * time.Second,
MaxEjectionTime: 300 * time.Second,
BaseEjectionTime: iserviceconfig.Duration(30 * time.Second),
MaxEjectionTime: iserviceconfig.Duration(300 * time.Second),
MaxEjectionPercent: 10,
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name(),
Config: emptyChildConfig{},
},
@ -1173,8 +1168,8 @@ func (s) TestConcurrentOperations(t *testing.T) {
},
BalancerConfig: &LBConfig{
Interval: math.MaxInt64, // so the interval will never run unless called manually in test.
BaseEjectionTime: 30 * time.Second,
MaxEjectionTime: 300 * time.Second,
BaseEjectionTime: iserviceconfig.Duration(30 * time.Second),
MaxEjectionTime: iserviceconfig.Duration(300 * time.Second),
MaxEjectionPercent: 10,
SuccessRateEjection: &SuccessRateEjection{ // Have both Success Rate and Failure Percentage to step through all the interval timer code
StdevFactor: 500,
@ -1188,7 +1183,7 @@ func (s) TestConcurrentOperations(t *testing.T) {
MinimumHosts: 3,
RequestVolume: 3,
},
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name(),
Config: emptyChildConfig{},
},
@ -1311,7 +1306,7 @@ func (s) TestConcurrentOperations(t *testing.T) {
},
BalancerConfig: &LBConfig{
Interval: math.MaxInt64,
ChildPolicy: &internalserviceconfig.BalancerConfig{
ChildPolicy: &iserviceconfig.BalancerConfig{
Name: t.Name(),
Config: emptyChildConfig{},
},

12
xds/internal/balancer/outlierdetection/config.go
View File

@ -18,9 +18,7 @@
package outlierdetection
import (
"time"
internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
"google.golang.org/grpc/serviceconfig"
)
@ -128,15 +126,15 @@ type LBConfig struct {
// Interval is the time interval between ejection analysis sweeps. This can
// result in both new ejections as well as addresses being returned to
// service. Defaults to 10s.
Interval time.Duration `json:"interval,omitempty"`
Interval iserviceconfig.Duration `json:"interval,omitempty"`
// BaseEjectionTime is the base time that a host is ejected for. The real
// time is equal to the base time multiplied by the number of times the host
// has been ejected and is capped by MaxEjectionTime. Defaults to 30s.
BaseEjectionTime time.Duration `json:"baseEjectionTime,omitempty"`
BaseEjectionTime iserviceconfig.Duration `json:"baseEjectionTime,omitempty"`
// MaxEjectionTime is the maximum time that an address is ejected for. If
// not specified, the default value (300s) or the BaseEjectionTime value is
// applied, whichever is larger.
MaxEjectionTime time.Duration `json:"maxEjectionTime,omitempty"`
MaxEjectionTime iserviceconfig.Duration `json:"maxEjectionTime,omitempty"`
// MaxEjectionPercent is the maximum % of an upstream cluster that can be
// ejected due to outlier detection. Defaults to 10% but will eject at least
// one host regardless of the value.
@ -148,7 +146,7 @@ type LBConfig struct {
// algorithm. If set, failure rate ejections will be performed.
FailurePercentageEjection *FailurePercentageEjection `json:"failurePercentageEjection,omitempty"`
// ChildPolicy is the config for the child policy.
ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
ChildPolicy *iserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
}
// EqualIgnoringChildPolicy returns whether the LBConfig is same with the

24
xds/internal/balancer/outlierdetection/e2e_test/outlierdetection_test.go
View File

@ -159,9 +159,9 @@ func (s) TestOutlierDetectionAlgorithmsE2E(t *testing.T) {
"loadBalancingConfig": [
{
"outlier_detection_experimental": {
"interval": 50000000,
"baseEjectionTime": 100000000,
"maxEjectionTime": 300000000000,
"interval": "0.050s",
"baseEjectionTime": "0.100s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 33,
"successRateEjection": {
"stdevFactor": 50,
@ -182,9 +182,9 @@ func (s) TestOutlierDetectionAlgorithmsE2E(t *testing.T) {
"loadBalancingConfig": [
{
"outlier_detection_experimental": {
"interval": 50000000,
"baseEjectionTime": 100000000,
"maxEjectionTime": 300000000000,
"interval": "0.050s",
"baseEjectionTime": "0.100s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 33,
"failurePercentageEjection": {
"threshold": 50,
@ -277,9 +277,9 @@ func (s) TestNoopConfiguration(t *testing.T) {
"loadBalancingConfig": [
{
"outlier_detection_experimental": {
"interval": 50000000,
"baseEjectionTime": 100000000,
"maxEjectionTime": 300000000000,
"interval": "0.050s",
"baseEjectionTime": "0.100s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 33,
"childPolicy": [{"round_robin": {}}]
}
@ -325,9 +325,9 @@ func (s) TestNoopConfiguration(t *testing.T) {
"loadBalancingConfig": [
{
"outlier_detection_experimental": {
"interval": 50000000,
"baseEjectionTime": 100000000,
"maxEjectionTime": 300000000000,
"interval": "0.050s",
"baseEjectionTime": "0.100s",
"maxEjectionTime": "300s",
"maxEjectionPercent": 33,
"failurePercentageEjection": {
"threshold": 50,