knative
/
pkg
mirror of https://github.com/knative/pkg.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make the chaosduck jitter (#1527)

This commit is contained in:
Matt Moore 2020-07-21 08:53:36 -07:00 committed by GitHub
parent 3b7ca76a63
commit c0a9ec7f11
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 25 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
leaderelection/chaosduck/main.go
View File

@ -30,6 +30,7 @@ import (
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
kubeclient "knative.dev/pkg/client/injection/kube/client" kubeclient "knative.dev/pkg/client/injection/kube/client"
"knative.dev/pkg/controller" "knative.dev/pkg/controller"
@ -45,13 +46,15 @@ type components map[string]sets.String
var ( var (
disabledComponents kflag.StringSet disabledComponents kflag.StringSet
tributePeriod time.Duration = 30 * time.Second tributePeriod time.Duration = 20 * time.Second
tributeFactor = 2.0
) )
func init() { func init() {
// Note that we don't explicitly call flag.Parse() because ParseAndGetConfigOrDie below does this already. // Note that we don't explicitly call flag.Parse() because ParseAndGetConfigOrDie below does this already.
flag.Var(&disabledComponents, "disable", "A repeatable flag to disable chaos for certain components.") flag.Var(&disabledComponents, "disable", "A repeatable flag to disable chaos for certain components.")
flag.DurationVar(&tributePeriod, "period", tributePeriod, "How frequently to terminate a leader pod per component.") flag.DurationVar(&tributePeriod, "period", tributePeriod, "How frequently to terminate a leader pod per component (this is the base duration used with the jitter factor from -factor).")
flag.Float64Var(&tributeFactor, "factor", tributeFactor, "The jitter factor to apply to the period.")
} }
func countingRFind(wr rune, wc int) func(rune) bool { func countingRFind(wr rune, wc int) func(rune) bool {
@ -126,31 +129,25 @@ func main() {
// Until we are shutdown, build up an index of components and kill // Until we are shutdown, build up an index of components and kill
// of a leader at the specified frequency. // of a leader at the specified frequency.
for { wait.JitterUntilWithContext(ctx, func(ctx context.Context) {
select { components, err := buildComponents(kc)
case <-time.After(tributePeriod): if err != nil {
components, err := buildComponents(kc) log.Printf("Error building components: %v", err)
if err != nil {
log.Printf("Error building components: %v", err)
}
log.Printf("Got components: %#v", components)
eg, ctx := errgroup.WithContext(ctx)
for name, leaders := range components {
if disabledComponents.Value.Has(name) {
continue
}
name, leaders := name, leaders
eg.Go(func() error {
return quack(ctx, kc, name, leaders)
})
}
if err := eg.Wait(); err != nil {
log.Printf("Ended iteration with err: %v", err)
}
case <-ctx.Done():
return
} }
} log.Printf("Got components: %#v", components)
eg, ctx := errgroup.WithContext(ctx)
for name, leaders := range components {
if disabledComponents.Value.Has(name) {
continue
}
name, leaders := name, leaders
eg.Go(func() error {
return quack(ctx, kc, name, leaders)
})
}
if err := eg.Wait(); err != nil {
log.Printf("Ended iteration with err: %v", err)
}
}, tributePeriod, tributeFactor, true /* sliding: do not include the runtime of the above in the interval */)
} }