From 560cba356baf83f7bdf58470f78d6bd335978971 Mon Sep 17 00:00:00 2001 From: calvin Date: Wed, 12 Apr 2023 19:17:42 +0800 Subject: [PATCH] optimize karmada aggregated apiserver startup Signed-off-by: calvin --- operator/pkg/init.go | 2 +- operator/pkg/tasks/init/apiserver.go | 56 +++++++++++++++++++++++++--- operator/pkg/tasks/init/etcd.go | 46 +++++++++++++++++++++-- operator/pkg/tasks/init/wait.go | 30 ++++----------- 4 files changed, 102 insertions(+), 32 deletions(-) diff --git a/operator/pkg/init.go b/operator/pkg/init.go index 9bab4c7f3..99590b219 100644 --- a/operator/pkg/init.go +++ b/operator/pkg/init.go @@ -75,7 +75,7 @@ func NewInitJob(opt *InitOptions) *workflow.Job { initJob.AppendTask(tasks.NewUploadCertsTask()) initJob.AppendTask(tasks.NewEtcdTask()) initJob.AppendTask(tasks.NewKarmadaApiserverTask()) - initJob.AppendTask(tasks.NewWaitApiserverTask()) + initJob.AppendTask(tasks.NewCheckApiserverHealthTask()) initJob.AppendTask(tasks.NewKarmadaResourcesTask()) initJob.AppendTask(tasks.NewComponentTask()) initJob.AppendTask(tasks.NewWaitControlPlaneTask()) diff --git a/operator/pkg/tasks/init/apiserver.go b/operator/pkg/tasks/init/apiserver.go index 92b17f737..d9fbe5579 100644 --- a/operator/pkg/tasks/init/apiserver.go +++ b/operator/pkg/tasks/init/apiserver.go @@ -3,11 +3,13 @@ package tasks import ( "errors" "fmt" + "time" "k8s.io/klog/v2" "github.com/karmada-io/karmada/operator/pkg/constants" "github.com/karmada-io/karmada/operator/pkg/controlplane/apiserver" + "github.com/karmada-io/karmada/operator/pkg/util/apiclient" "github.com/karmada-io/karmada/operator/pkg/workflow" ) @@ -23,10 +25,18 @@ func NewKarmadaApiserverTask() workflow.Task { Name: constants.KarmadaAPIserverComponent, Run: runKarmadaAPIServer, }, + { + Name: fmt.Sprintf("%s-%s", "wait", constants.KarmadaAPIserverComponent), + Run: runWaitKarmadaAPIServer, + }, { Name: constants.KarmadaAggregratedAPIServerComponent, Run: runKarmadaAggregratedAPIServer, }, + { + Name: fmt.Sprintf("%s-%s", "wait", constants.KarmadaAggregratedAPIServerComponent), + Run: runWaitKarmadaAggregratedAPIServer, + }, }, } } @@ -44,12 +54,12 @@ func runApiserver(r workflow.RunData) error { func runKarmadaAPIServer(r workflow.RunData) error { data, ok := r.(InitData) if !ok { - return errors.New("karmadaApiserver task invoked with an invalid data struct") + return errors.New("KarmadaApiserver task invoked with an invalid data struct") } cfg := data.Components() if cfg.KarmadaAPIServer == nil { - klog.V(2).InfoS("[karmadaApiserver] Skip install karmada-apiserver component") + klog.V(2).InfoS("[KarmadaApiserver] Skip install karmada-apiserver component") return nil } @@ -58,19 +68,36 @@ func runKarmadaAPIServer(r workflow.RunData) error { return fmt.Errorf("failed to install karmada apiserver component, err: %w", err) } - klog.V(2).InfoS("[karmadaApiserver] Successfully installed apiserver component", "karmada", klog.KObj(data)) + klog.V(2).InfoS("[KarmadaApiserver] Successfully installed karmada-apiserver component", "karmada", klog.KObj(data)) + return nil +} + +func runWaitKarmadaAPIServer(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("wait-KarmadaAPIServer task invoked with an invalid data struct") + } + + waiter := apiclient.NewKarmadaWaiter(data.ControlplaneConifg(), data.RemoteClient(), time.Second*30) + + err := waiter.WaitForSomePods(karmadaApiserverLabels.String(), data.GetNamespace(), 1) + if err != nil { + return fmt.Errorf("waiting for karmada-apiserver to ready timeout, err: %w", err) + } + + klog.V(2).InfoS("[wait-KarmadaAPIServer] the karmada-apiserver is ready", "karmada", klog.KObj(data)) return nil } func runKarmadaAggregratedAPIServer(r workflow.RunData) error { data, ok := r.(InitData) if !ok { - return errors.New("karmadaAggregratedApiServer task invoked with an invalid data struct") + return errors.New("KarmadaAggregratedAPIServer task invoked with an invalid data struct") } cfg := data.Components() if cfg.KarmadaAggregratedAPIServer == nil { - klog.V(2).InfoS("[KarmadaAggregratedApiServer] Skip install karmada-aggregrated-apiserver component") + klog.V(2).InfoS("[KarmadaAggregratedAPIServer] Skip install karmada-aggregrated-apiserver component") return nil } @@ -79,6 +106,23 @@ func runKarmadaAggregratedAPIServer(r workflow.RunData) error { return fmt.Errorf("failed to install karmada aggregrated apiserver, err: %w", err) } - klog.V(2).InfoS("[KarmadaAggregratedApiServer] Successfully installed karmada apiserve component", "karmada", klog.KObj(data)) + klog.V(2).InfoS("[KarmadaAggregratedApiserve] Successfully installed karmada-aggregrated-apiserver component", "karmada", klog.KObj(data)) + return nil +} + +func runWaitKarmadaAggregratedAPIServer(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("wait-KarmadaAggregratedAPIServer task invoked with an invalid data struct") + } + + waiter := apiclient.NewKarmadaWaiter(data.ControlplaneConifg(), data.RemoteClient(), time.Second*30) + + err := waiter.WaitForSomePods(karmadaAggregatedAPIServerLabels.String(), data.GetNamespace(), 1) + if err != nil { + return fmt.Errorf("waiting for karmada-apiserver to ready timeout, err: %w", err) + } + + klog.V(2).InfoS("[wait-KarmadaAggregratedAPIServer] the karmada-aggregated-apiserver is ready", "karmada", klog.KObj(data)) return nil } diff --git a/operator/pkg/tasks/init/etcd.go b/operator/pkg/tasks/init/etcd.go index 6261b7430..98498f00a 100644 --- a/operator/pkg/tasks/init/etcd.go +++ b/operator/pkg/tasks/init/etcd.go @@ -3,18 +3,31 @@ package tasks import ( "errors" "fmt" + "time" "k8s.io/klog/v2" "github.com/karmada-io/karmada/operator/pkg/controlplane/etcd" + "github.com/karmada-io/karmada/operator/pkg/util/apiclient" "github.com/karmada-io/karmada/operator/pkg/workflow" ) // NewEtcdTask init a etcd task to install etcd component func NewEtcdTask() workflow.Task { return workflow.Task{ - Name: "Etcd", - Run: runEtcd, + Name: "Etcd", + Run: runEtcd, + RunSubTasks: true, + Tasks: []workflow.Task{ + { + Name: "deploy-etcd", + Run: runDeployEtcd, + }, + { + Name: "wait-etcd", + Run: runWaitEtcd, + }, + }, } } @@ -23,7 +36,16 @@ func runEtcd(r workflow.RunData) error { if !ok { return errors.New("etcd task invoked with an invalid data struct") } + klog.V(4).InfoS("[etcd] Running etcd task", "karmada", klog.KObj(data)) + return nil +} + +func runDeployEtcd(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("deploy-etcd task invoked with an invalid data struct") + } cfg := data.Components() if cfg.Etcd.External != nil { @@ -40,6 +62,24 @@ func runEtcd(r workflow.RunData) error { return fmt.Errorf("failed to install etcd component, err: %w", err) } - klog.V(2).InfoS("[etcd] Successfully installed etcd component", "karmada", klog.KObj(data)) + klog.V(2).InfoS("[deploy-etcd] Successfully installed etcd component", "karmada", klog.KObj(data)) + return nil +} + +func runWaitEtcd(r workflow.RunData) error { + data, ok := r.(InitData) + if !ok { + return errors.New("wait-etcd task invoked with an invalid data struct") + } + + waiter := apiclient.NewKarmadaWaiter(data.ControlplaneConifg(), data.RemoteClient(), time.Second*30) + + // wait etcd, karmada apiserver and aggregated apiserver to ready + // as long as a replica of pod is ready, we consider the service available. + if err := waiter.WaitForSomePods(etcdLabels.String(), data.GetNamespace(), 1); err != nil { + return fmt.Errorf("waiting for etcd to ready timeout, err: %w", err) + } + + klog.V(2).InfoS("[wait-etcd] the etcd pods is ready", "karmada", klog.KObj(data)) return nil } diff --git a/operator/pkg/tasks/init/wait.go b/operator/pkg/tasks/init/wait.go index c61257740..26f41e5fd 100644 --- a/operator/pkg/tasks/init/wait.go +++ b/operator/pkg/tasks/init/wait.go @@ -23,10 +23,10 @@ var ( karmadaWebhookLabels = labels.Set{"karmada-app": constants.KarmadaWebhook} ) -// NewWaitApiserverTask init wait-apiserver task -func NewWaitApiserverTask() workflow.Task { +// NewCheckApiserverHealthTask init wait-apiserver task +func NewCheckApiserverHealthTask() workflow.Task { return workflow.Task{ - Name: "wait-apiserver", + Name: "check-apiserver-health", Run: runWaitApiserver, } } @@ -34,31 +34,17 @@ func NewWaitApiserverTask() workflow.Task { func runWaitApiserver(r workflow.RunData) error { data, ok := r.(InitData) if !ok { - return fmt.Errorf("wait-aipserver task invoked with an invalid data struct") + return fmt.Errorf("check-apiserver-health task invoked with an invalid data struct") } - klog.V(4).InfoS("[wait-aipserver] Running task", "karmada", klog.KObj(data)) + klog.V(4).InfoS("[check-apiserver-health] Running task", "karmada", klog.KObj(data)) waiter := apiclient.NewKarmadaWaiter(data.ControlplaneConifg(), data.RemoteClient(), time.Second*30) - // wait etcd, karmada apiserver and aggregated apiserver to ready - // as long as a replica of pod is ready, we consider the service available. - if err := waiter.WaitForSomePods(etcdLabels.String(), data.GetNamespace(), 1); err != nil { - return fmt.Errorf("waiting for etcd to ready timeout, err: %w", err) - } - if err := waiter.WaitForSomePods(karmadaApiserverLabels.String(), data.GetNamespace(), 1); err != nil { - return fmt.Errorf("waiting for karmada apiserver to ready timeout, err: %w", err) - } - err := waiter.WaitForSomePods(karmadaAggregatedAPIServerLabels.String(), data.GetNamespace(), 1) - if err != nil { - return fmt.Errorf("waiting for karmada aggregated apiserver to ready timeout, err: %w", err) - } - - // check whether the karmada apiserver is running and health. + // check whether the karmada apiserver is health. if err := apiclient.TryRunCommand(waiter.WaitForAPI, 3); err != nil { - return fmt.Errorf("the karmada apiserver is unhealth, err: %w", err) + return fmt.Errorf("the karmada apiserver is unhealthy, err: %w", err) } - - klog.V(2).InfoS("[wait-aipserver] the etcd, karmada apiserver and aggregated apiserver is ready", "karmada", klog.KObj(data)) + klog.V(2).InfoS("[check-apiserver-health] the etcd and karmada-apiserver is healthy", "karmada", klog.KObj(data)) return nil }