karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3221 from yanfeng1992/wait-dep-sts-rollout 

check deployment statefulset rollout not use check pod ready
This commit is contained in:
karmada-bot 2023-03-16 17:39:20 +08:00 committed by GitHub
parent fb974eeacf fd062cecab
commit 7f5bb9dde6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 112 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/karmadactl/addons/descheduler/descheduler.go
View File

@ -13,13 +13,9 @@ import (
addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init"
addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils"
"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes"
initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils"
cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
)
var karmadaDeschedulerLabels = map[string]string{"app": addoninit.DeschedulerResourceName}
// AddonDescheduler describe the descheduler addon command process
var AddonDescheduler = &addoninit.Addon{
Name: addoninit.DeschedulerResourceName,
@ -64,7 +60,7 @@ var enableDescheduler = func(opts *addoninit.CommandAddonsEnableOption) error {
return fmt.Errorf("create karmada descheduler deployment error: %v", err)
}
if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaDeschedulerLabels), opts.WaitPodReadyTimeout); err != nil {
if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaDeschedulerDeployment, opts.WaitComponentReadyTimeout); err != nil {
return fmt.Errorf("wait karmada descheduler pod timeout: %v", err)
}

3
pkg/karmadactl/addons/enable.go
View File

@ -8,6 +8,7 @@ import (
"k8s.io/kubectl/pkg/util/templates"
addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init"
"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options"
"github.com/karmada-io/karmada/pkg/version"
)
@ -71,7 +72,7 @@ func NewCmdAddonsEnable(parentCommand string) *cobra.Command {
flags := cmd.PersistentFlags()
opts.GlobalCommandOptions.AddFlags(flags)
flags.IntVar(&opts.WaitPodReadyTimeout, "pod-timeout", 30, "Wait pod ready timeout.")
flags.IntVar(&opts.WaitComponentReadyTimeout, "pod-timeout", options.WaitComponentReadyTimeout, "Wait pod ready timeout.")
flags.IntVar(&opts.WaitAPIServiceReadyTimeout, "apiservice-timeout", 30, "Wait apiservice ready timeout.")
flags.StringVar(&opts.KarmadaSearchImage, "karmada-search-image", fmt.Sprintf("docker.io/karmada/karmada-search:%s", releaseVer.PatchRelease()), "karmada search image")
flags.Int32Var(&opts.KarmadaSearchReplicas, "karmada-search-replicas", 1, "Karmada search replica set")

5
pkg/karmadactl/addons/estimator/estimator.go
View File

@ -15,8 +15,6 @@ import (
addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init"
addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils"
"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes"
initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils"
cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
"github.com/karmada-io/karmada/pkg/util/names"
)
@ -113,8 +111,7 @@ var enableEstimator = func(opts *addoninit.CommandAddonsEnableOption) error {
return fmt.Errorf("create or update scheduler estimator deployment error: %v", err)
}
karmadaEstimatorLabels := map[string]string{"cluster": opts.Cluster}
if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaEstimatorLabels), opts.WaitPodReadyTimeout); err != nil {
if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaEstimatorDeployment, opts.WaitComponentReadyTimeout); err != nil {
klog.Warning(err)
}
klog.Infof("Karmada scheduler estimator of member cluster %s is installed successfully.", opts.Cluster)

2
pkg/karmadactl/addons/init/disable_option.go
View File

@ -15,8 +15,6 @@ type CommandAddonsDisableOption struct {
GlobalCommandOptions
KarmadaKubeClientSet *kubernetes.Clientset
WaitPodReadyTimeout int
}
// Complete the conditions required to be able to run disable.

2
pkg/karmadactl/addons/init/enable_option.go
View File

@ -32,7 +32,7 @@ type CommandAddonsEnableOption struct {
KarmadaKubeClientSet *kubernetes.Clientset
WaitPodReadyTimeout int
WaitComponentReadyTimeout int
WaitAPIServiceReadyTimeout int

8
pkg/karmadactl/addons/search/search.go
View File

@ -19,8 +19,6 @@ import (
addoninit "github.com/karmada-io/karmada/pkg/karmadactl/addons/init"
addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils"
initkarmada "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/karmada"
"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes"
initutils "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils"
cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
)
@ -35,10 +33,6 @@ const (
etcdContainerClientPort = 2379
)
var (
karmadaSearchLabels = map[string]string{"app": addoninit.SearchResourceName, "apiserver": "true"}
)
// AddonSearch describe the search addon command process
var AddonSearch = &addoninit.Addon{
Name: addoninit.SearchResourceName,
@ -166,7 +160,7 @@ func installComponentsOnHostCluster(opts *addoninit.CommandAddonsEnableOption) e
return fmt.Errorf("create karmada search deployment error: %v", err)
}
if err := kubernetes.WaitPodReady(opts.KubeClientSet, opts.Namespace, initutils.MapToString(karmadaSearchLabels), opts.WaitPodReadyTimeout); err != nil {
if err := cmdutil.WaitForDeploymentRollout(opts.KubeClientSet, karmadaSearchDeployment, opts.WaitComponentReadyTimeout); err != nil {
return fmt.Errorf("wait karmada search pod status ready timeout: %v", err)
}

107
pkg/karmadactl/cmdinit/kubernetes/check.go
View File

@ -1,107 +0,0 @@
package kubernetes
import (
"context"
"fmt"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
)
func podStatus(pod *corev1.Pod) string {
for _, value := range pod.Status.ContainerStatuses {
if pod.Status.Phase == corev1.PodRunning {
if value.State.Waiting != nil {
return value.State.Waiting.Reason
}
if value.State.Waiting == nil {
return string(corev1.PodRunning)
}
return "Error"
}
if pod.ObjectMeta.DeletionTimestamp != nil {
return "Terminating"
}
}
return pod.Status.ContainerStatuses[0].State.Waiting.Reason
}
func isPodReady(c kubernetes.Interface, n, p string) wait.ConditionFunc {
return func() (done bool, err error) {
pod, err := c.CoreV1().Pods(n).Get(context.TODO(), p, metav1.GetOptions{})
if err != nil {
return false, err
}
if pod.Status.Phase == corev1.PodPending && len(pod.Status.ContainerStatuses) == 0 {
klog.Warningf("Pod: %s not ready. status: %v", pod.Name, corev1.PodPending)
return false, nil
}
for _, v := range pod.Status.Conditions {
switch v.Type {
case corev1.PodReady:
if v.Status == corev1.ConditionTrue {
klog.Infof("pod: %s is ready. status: %v", pod.Name, podStatus(pod))
return true, nil
}
klog.Warningf("Pod: %s not ready. status: %v", pod.Name, podStatus(pod))
return false, nil
default:
continue
}
}
return false, err
}
}
// waitPodReady Poll up to timeout seconds for pod to enter running state.
// Returns an error if the pod never enters the running state.
func waitPodReady(c kubernetes.Interface, namespaces, podName string, timeout time.Duration) error {
return wait.PollImmediate(time.Second, timeout, isPodReady(c, namespaces, podName))
}
// WaitPodReady wait pod ready
func WaitPodReady(c kubernetes.Interface, namespace, selector string, timeout int) error {
// Wait 3 second
time.Sleep(3 * time.Second)
pods, err := c.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector})
if err != nil {
return err
}
if len(pods.Items) == 0 {
return fmt.Errorf("no pods in %s with selector %s", namespace, selector)
}
for _, pod := range pods.Items {
if err = waitPodReady(c, namespace, pod.Name, time.Duration(timeout)*time.Second); err != nil {
return err
}
}
return nil
}
// WaitEtcdReplicasetInDesired Wait Etcd Ready
func WaitEtcdReplicasetInDesired(replicas int32, c kubernetes.Interface, namespace, selector string, timeout int) error {
if err := wait.PollImmediate(time.Second, time.Duration(timeout)*time.Second, func() (done bool, err error) {
pods, e := c.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: selector})
if e != nil {
return false, nil
}
if int32(len(pods.Items)) == replicas {
klog.Infof("Etcd desired replicaset is %v, currently: %v", replicas, len(pods.Items))
return true, nil
}
klog.Warningf("Etcd desired replicaset is %v, currently: %v", replicas, len(pods.Items))
return false, nil
}); err != nil {
return err
}
return nil
}

44
pkg/karmadactl/cmdinit/kubernetes/deploy.go
View File

@ -348,28 +348,30 @@ func (i *CommandInitOption) createCertsSecrets() error {
}
func (i *CommandInitOption) initKarmadaAPIServer() error {
// wait karmada APIServer component ready timeout 120s
waitKarmadaAPIServerComponentReadyTimeout := 120
if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeEtcdService(etcdStatefulSetAndServiceName)); err != nil {
return err
}
klog.Info("Create etcd StatefulSets")
if _, err := i.KubeClientSet.AppsV1().StatefulSets(i.Namespace).Create(context.TODO(), i.makeETCDStatefulSet(), metav1.CreateOptions{}); err != nil {
etcdStatefulSet := i.makeETCDStatefulSet()
if _, err := i.KubeClientSet.AppsV1().StatefulSets(i.Namespace).Create(context.TODO(), etcdStatefulSet, metav1.CreateOptions{}); err != nil {
klog.Warning(err)
}
if err := WaitEtcdReplicasetInDesired(i.EtcdReplicas, i.KubeClientSet, i.Namespace, utils.MapToString(etcdLabels), 30); err != nil {
if err := util.WaitForStatefulSetRollout(i.KubeClientSet, etcdStatefulSet, options.WaitComponentReadyTimeout); err != nil {
klog.Warning(err)
}
if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(etcdLabels), 30); err != nil {
klog.Warning(err)
}
klog.Info("Create karmada ApiServer Deployment")
if err := util.CreateOrUpdateService(i.KubeClientSet, i.makeKarmadaAPIServerService()); err != nil {
return err
}
if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAPIServerDeployment(), metav1.CreateOptions{}); err != nil {
karmadaAPIServerDeployment := i.makeKarmadaAPIServerDeployment()
if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), karmadaAPIServerDeployment, metav1.CreateOptions{}); err != nil {
klog.Warning(err)
}
if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(apiServerLabels), 120); err != nil {
if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaAPIServerDeployment, waitKarmadaAPIServerComponentReadyTimeout); err != nil {
return err
}
@ -379,19 +381,17 @@ func (i *CommandInitOption) initKarmadaAPIServer() error {
if err := util.CreateOrUpdateService(i.KubeClientSet, i.karmadaAggregatedAPIServerService()); err != nil {
klog.Exitln(err)
}
karmadaAggregatedAPIServerDeployment := i.makeKarmadaAggregatedAPIServerDeployment()
if _, err := i.KubeClientSet.AppsV1().Deployments(i.Namespace).Create(context.TODO(), i.makeKarmadaAggregatedAPIServerDeployment(), metav1.CreateOptions{}); err != nil {
klog.Warning(err)
}
if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(aggregatedAPIServerLabels), 30); err != nil {
if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaAggregatedAPIServerDeployment, options.WaitComponentReadyTimeout); err != nil {
klog.Warning(err)
}
return nil
}
func (i *CommandInitOption) initKarmadaComponent() error {
// wait pod ready timeout 30s
waitPodReadyTimeout := 30
deploymentClient := i.KubeClientSet.AppsV1().Deployments(i.Namespace)
// Create karmada-kube-controller-manager
// https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/kube-controller-manager.yaml
@ -399,30 +399,33 @@ func (i *CommandInitOption) initKarmadaComponent() error {
if err := util.CreateOrUpdateService(i.KubeClientSet, i.kubeControllerManagerService()); err != nil {
klog.Exitln(err)
}
if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaKubeControllerManagerDeployment(), metav1.CreateOptions{}); err != nil {
karmadaKubeControllerManagerDeployment := i.makeKarmadaKubeControllerManagerDeployment()
if _, err := deploymentClient.Create(context.TODO(), karmadaKubeControllerManagerDeployment, metav1.CreateOptions{}); err != nil {
klog.Warning(err)
}
if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(kubeControllerManagerLabels), waitPodReadyTimeout); err != nil {
if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaKubeControllerManagerDeployment, options.WaitComponentReadyTimeout); err != nil {
klog.Warning(err)
}
// Create karmada-scheduler
// https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/karmada-scheduler.yaml
klog.Info("Create karmada scheduler Deployment")
if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaSchedulerDeployment(), metav1.CreateOptions{}); err != nil {
karmadaSchedulerDeployment := i.makeKarmadaSchedulerDeployment()
if _, err := deploymentClient.Create(context.TODO(), karmadaSchedulerDeployment, metav1.CreateOptions{}); err != nil {
klog.Warning(err)
}
if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(schedulerLabels), waitPodReadyTimeout); err != nil {
if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaSchedulerDeployment, options.WaitComponentReadyTimeout); err != nil {
klog.Warning(err)
}
// Create karmada-controller-manager
// https://github.com/karmada-io/karmada/blob/master/artifacts/deploy/karmada-controller-manager.yaml
klog.Info("Create karmada controller manager Deployment")
if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaControllerManagerDeployment(), metav1.CreateOptions{}); err != nil {
karmadaControllerManagerDeployment := i.makeKarmadaControllerManagerDeployment()
if _, err := deploymentClient.Create(context.TODO(), karmadaControllerManagerDeployment, metav1.CreateOptions{}); err != nil {
klog.Warning(err)
}
if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(controllerManagerLabels), waitPodReadyTimeout); err != nil {
if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaControllerManagerDeployment, options.WaitComponentReadyTimeout); err != nil {
klog.Warning(err)
}
@ -432,10 +435,11 @@ func (i *CommandInitOption) initKarmadaComponent() error {
if err := util.CreateOrUpdateService(i.KubeClientSet, i.karmadaWebhookService()); err != nil {
klog.Exitln(err)
}
if _, err := deploymentClient.Create(context.TODO(), i.makeKarmadaWebhookDeployment(), metav1.CreateOptions{}); err != nil {
karmadaWebhookDeployment := i.makeKarmadaWebhookDeployment()
if _, err := deploymentClient.Create(context.TODO(), karmadaWebhookDeployment, metav1.CreateOptions{}); err != nil {
klog.Warning(err)
}
if err := WaitPodReady(i.KubeClientSet, i.Namespace, utils.MapToString(webhookLabels), waitPodReadyTimeout); err != nil {
if err := util.WaitForDeploymentRollout(i.KubeClientSet, karmadaWebhookDeployment, options.WaitComponentReadyTimeout); err != nil {
klog.Warning(err)
}
return nil

2
pkg/karmadactl/cmdinit/options/global.go
View File

@ -23,4 +23,6 @@ const (
UserName = "karmada-admin"
// KarmadaKubeConfigName karmada kubeconfig name
KarmadaKubeConfigName = "karmada-apiserver.config"
// WaitComponentReadyTimeout wait component ready time
WaitComponentReadyTimeout = 30
)

7
pkg/karmadactl/register/register.go
View File

@ -34,8 +34,6 @@ import (
"k8s.io/kubectl/pkg/util/templates"
"github.com/karmada-io/karmada/pkg/apis/cluster/validation"
check "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/kubernetes"
"github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils"
"github.com/karmada-io/karmada/pkg/karmadactl/options"
cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
"github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient"
@ -349,11 +347,12 @@ func (o *CommandRegisterOption) Run(parentCommand string) error {
// create karmada-agent Deployment in the member cluster
fmt.Println("[karmada-agent-start] Waiting karmada-agent Deployment")
if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), o.makeKarmadaAgentDeployment(), metav1.CreateOptions{}); err != nil {
KarmadaAgentDeployment := o.makeKarmadaAgentDeployment()
if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil {
return err
}
if err := check.WaitPodReady(o.memberClusterClient, o.Namespace, utils.MapToString(karmadaAgentLabels), int(o.Timeout)); err != nil {
if err := cmdutil.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil {
return err
}

77
pkg/karmadactl/util/check.go Normal file
View File

@ -0,0 +1,77 @@
package util
import (
"context"
"fmt"
"time"
appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
)
// WaitForStatefulSetRollout wait for StatefulSet reaches the ready state or timeout.
func WaitForStatefulSetRollout(c kubernetes.Interface, sts *appsv1.StatefulSet, timeoutSeconds int) error {
var lastErr error
pollError := wait.PollImmediate(time.Second, time.Duration(timeoutSeconds)*time.Second, func() (bool, error) {
s, err := c.AppsV1().StatefulSets(sts.GetNamespace()).Get(context.TODO(), sts.GetName(), metav1.GetOptions{})
if err != nil {
lastErr = err
return false, nil
}
if s.Generation != s.Status.ObservedGeneration {
lastErr = fmt.Errorf("expected generation %d, observed generation: %d",
s.Generation, s.Status.ObservedGeneration)
return false, nil
}
if (s.Spec.Replicas != nil) && (s.Status.UpdatedReplicas < *s.Spec.Replicas) {
lastErr = fmt.Errorf("expected %d replicas, got %d updated replicas",
*s.Spec.Replicas, s.Status.UpdatedReplicas)
return false, nil
}
if s.Status.AvailableReplicas < s.Status.UpdatedReplicas {
lastErr = fmt.Errorf("expected %d replicas, got %d available replicas",
s.Status.UpdatedReplicas, s.Status.AvailableReplicas)
return false, nil
}
return true, nil
})
if pollError != nil {
return fmt.Errorf("wait for Statefulset(%s/%s) rollout: %v: %v", sts.GetNamespace(), sts.GetName(), pollError, lastErr)
}
return nil
}
// WaitForDeploymentRollout wait for Deployment reaches the ready state or timeout.
func WaitForDeploymentRollout(c kubernetes.Interface, dep *appsv1.Deployment, timeoutSeconds int) error {
var lastErr error
pollError := wait.PollImmediate(time.Second, time.Duration(timeoutSeconds)*time.Second, func() (bool, error) {
d, err := c.AppsV1().Deployments(dep.GetNamespace()).Get(context.TODO(), dep.GetName(), metav1.GetOptions{})
if err != nil {
lastErr = err
return false, nil
}
if d.Generation != d.Status.ObservedGeneration {
lastErr = fmt.Errorf("current generation %d, observed generation %d",
d.Generation, d.Status.ObservedGeneration)
return false, nil
}
if (d.Spec.Replicas != nil) && (d.Status.UpdatedReplicas < *d.Spec.Replicas) {
lastErr = fmt.Errorf("the number of pods targeted by the deployment (%d pods) is different "+
"from the number of pods targeted by the deployment that have the desired template spec (%d pods)",
*d.Spec.Replicas, d.Status.UpdatedReplicas)
return false, nil
}
if d.Status.AvailableReplicas < d.Status.UpdatedReplicas {
lastErr = fmt.Errorf("expected %d replicas, got %d available replicas",
d.Status.UpdatedReplicas, d.Status.AvailableReplicas)
return false, nil
}
return true, nil
})
if pollError != nil {
return fmt.Errorf("wait for Deployment(%s/%s) rollout: %v: %v", dep.GetNamespace(), dep.GetName(), pollError, lastErr)
}
return nil
}