From 4ebc27a549c082f8ca83d87ceb23c86dfe9766bc Mon Sep 17 00:00:00 2001 From: Andrea Mazzotti Date: Mon, 7 Apr 2025 11:15:04 +0200 Subject: [PATCH] Add e2e test to prevent Machine rollout on provider upgrade Signed-off-by: Andrea Mazzotti --- test/e2e/e2e_upgrade_test.go | 55 ++++++++++++++++++++++++++++++++ test/e2e/helpers.go | 61 ++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/test/e2e/e2e_upgrade_test.go b/test/e2e/e2e_upgrade_test.go index 7ad0187..caca143 100644 --- a/test/e2e/e2e_upgrade_test.go +++ b/test/e2e/e2e_upgrade_test.go @@ -24,6 +24,8 @@ import ( "fmt" "os" "path/filepath" + "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -114,6 +116,24 @@ var _ = Describe("Workload cluster creation", func() { ControlPlane: client.ObjectKeyFromObject(result.ControlPlane), }, e2eConfig.GetIntervals(specName, "wait-control-plane")...) + WaitForClusterReady(ctx, WaitForClusterReadyInput{ + Getter: bootstrapClusterProxy.GetClient(), + Name: result.Cluster.Name, + Namespace: result.Cluster.Namespace, + }, e2eConfig.GetIntervals(specName, "wait-cluster")...) + + By("Fetching all Machines") + machineList := GetMachinesByCluster(ctx, GetMachinesByClusterInput{ + Lister: bootstrapClusterProxy.GetClient(), + ClusterName: result.Cluster.Name, + Namespace: result.Cluster.Namespace, + }) + Expect(machineList.Items).ShouldNot(BeEmpty(), "There must be at least one Machine") + machinesNames := []string{} + for _, machine := range machineList.Items { + machinesNames = append(machinesNames, machine.Name) + } + By("Upgrading to next boostrap/controlplane provider version") UpgradeManagementCluster(ctx, clusterctl.UpgradeManagementClusterAndWaitInput{ ClusterProxy: bootstrapClusterProxy, @@ -128,6 +148,41 @@ var _ = Describe("Workload cluster creation", func() { ControlPlane: client.ObjectKeyFromObject(result.ControlPlane), }, e2eConfig.GetIntervals(specName, "wait-control-plane")...) + WaitForClusterReady(ctx, WaitForClusterReadyInput{ + Getter: bootstrapClusterProxy.GetClient(), + Name: result.Cluster.Name, + Namespace: result.Cluster.Namespace, + }, e2eConfig.GetIntervals(specName, "wait-cluster")...) + + By("Verifying machine rollout did not happen") + Consistently(func() error { + updatedMachineList := GetMachinesByCluster(ctx, GetMachinesByClusterInput{ + Lister: bootstrapClusterProxy.GetClient(), + ClusterName: result.Cluster.Name, + Namespace: result.Cluster.Namespace, + }) + if len(updatedMachineList.Items) == 0 { + return fmt.Errorf("There must be at least one Machine after provider upgrade") + } + updatedMachinesNames := []string{} + for _, machine := range updatedMachineList.Items { + updatedMachinesNames = append(updatedMachinesNames, machine.Name) + } + sameMachines, err := ContainElements(machinesNames).Match(updatedMachinesNames) + if err != nil { + return fmt.Errorf("matching machines: %w", err) + } + if !sameMachines { + fmt.Printf("Pre-upgrade machines: [%s]\n", strings.Join(machinesNames, ",")) + fmt.Printf("Post-upgrade machines: [%s]\n", strings.Join(updatedMachinesNames, ",")) + return fmt.Errorf("Machines should not have been rolled out after provider upgrade") + } + if len(updatedMachinesNames) != len(machinesNames) { + return fmt.Errorf("Number of Machines '%d' should match after provider upgrade '%d'", len(machinesNames), len(updatedMachinesNames)) + } + return nil + }).WithTimeout(2 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + By("Scaling down control plane to 2 and workers up to 2") ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ ClusterProxy: bootstrapClusterProxy, diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 588df9e..dd8866a 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -36,6 +36,7 @@ import ( pkgerrors "github.com/pkg/errors" controlplanev1 "github.com/rancher/cluster-api-provider-rke2/controlplane/api/v1beta1" + corev1 "k8s.io/api/core/v1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" @@ -302,6 +303,30 @@ func GetRKE2ControlPlaneByCluster(ctx context.Context, input GetRKE2ControlPlane return nil } +// GetMachinesByClusterInput is the input for GetRKE2ControlPlaneByCluster. +type GetMachinesByClusterInput struct { + Lister framework.Lister + ClusterName string + Namespace string +} + +// GetMachinesByCluster returns the Machine objects for a cluster. +func GetMachinesByCluster(ctx context.Context, input GetMachinesByClusterInput) *clusterv1.MachineList { + opts := []client.ListOption{ + client.InNamespace(input.Namespace), + client.MatchingLabels{ + clusterv1.ClusterNameLabel: input.ClusterName, + }, + } + + machineList := &clusterv1.MachineList{} + Eventually(func() error { + return input.Lister.List(ctx, machineList, opts...) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list Machine objects for Cluster %s", klog.KRef(input.Namespace, input.ClusterName)) + + return machineList +} + // WaitForControlPlaneAndMachinesReadyInput is the input type for WaitForControlPlaneAndMachinesReady. type WaitForControlPlaneAndMachinesReadyInput struct { GetLister framework.GetLister @@ -488,6 +513,42 @@ func WaitForClusterToUpgrade(ctx context.Context, input WaitForClusterToUpgradeI }, intervals...).Should(Succeed()) } +// WaitForClusterReadyInput is the input type for WaitForClusterReady. +type WaitForClusterReadyInput struct { + Getter framework.Getter + Name string + Namespace string +} + +// WaitForClusterReady will wait for a Cluster to be Ready. +func WaitForClusterReady(ctx context.Context, input WaitForClusterReadyInput, intervals ...interface{}) { + By("Waiting for Cluster to be Ready") + + Eventually(func() error { + cluster := &clusterv1.Cluster{} + key := types.NamespacedName{Name: input.Name, Namespace: input.Namespace} + + if err := input.Getter.Get(ctx, key, cluster); err != nil { + return fmt.Errorf("getting Cluster %s/%s: %w", input.Namespace, input.Name, err) + } + + readyCondition := conditions.Get(cluster, clusterv1.ReadyCondition) + if readyCondition == nil { + return fmt.Errorf("Cluster Ready condition is not found") + } + + switch readyCondition.Status { + case corev1.ConditionTrue: + //Cluster is ready + return nil + case corev1.ConditionFalse: + return fmt.Errorf("Cluster is not Ready") + default: + return fmt.Errorf("Cluster Ready condition is unknown") + } + }, intervals...).Should(Succeed()) +} + // setDefaults sets the default values for ApplyCustomClusterTemplateAndWaitInput if not set. // Currently, we set the default ControlPlaneWaiters here, which are implemented for RKE2ControlPlane. func setDefaults(input *ApplyCustomClusterTemplateAndWaitInput) {