291 lines
12 KiB
Go
291 lines
12 KiB
Go
//go:build e2e
|
|
// +build e2e
|
|
|
|
/*
|
|
Copyright 2023 SUSE.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package e2e
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
"gopkg.in/yaml.v3"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/utils/ptr"
|
|
|
|
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
|
|
"sigs.k8s.io/cluster-api/util"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
)
|
|
|
|
var _ = Describe("Workload cluster creation", func() {
|
|
var (
|
|
specName = "create-workload-cluster"
|
|
namespace *corev1.Namespace
|
|
cancelWatches context.CancelFunc
|
|
result *ApplyClusterTemplateAndWaitResult
|
|
clusterName string
|
|
clusterctlLogFolder string
|
|
)
|
|
|
|
BeforeEach(func() {
|
|
Expect(e2eConfig).ToNot(BeNil(), "Invalid argument. e2eConfig can't be nil when calling %s spec", specName)
|
|
Expect(clusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. clusterctlConfigPath must be an existing file when calling %s spec", specName)
|
|
Expect(bootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. bootstrapClusterProxy can't be nil when calling %s spec", specName)
|
|
Expect(os.MkdirAll(artifactFolder, 0755)).To(Succeed(), "Invalid argument. artifactFolder can't be created for %s spec", specName)
|
|
|
|
Expect(e2eConfig.Variables).To(HaveKey(KubernetesVersion))
|
|
|
|
By("Initializing the bootstrap cluster")
|
|
initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder)
|
|
|
|
clusterName = fmt.Sprintf("caprke2-e2e-%s", util.RandomString(6))
|
|
|
|
// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
|
|
namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder)
|
|
|
|
result = new(ApplyClusterTemplateAndWaitResult)
|
|
|
|
// We need to override clusterctl apply log folder to avoid getting our credentials exposed.
|
|
clusterctlLogFolder = filepath.Join(os.TempDir(), "clusters", bootstrapClusterProxy.GetName())
|
|
})
|
|
|
|
AfterEach(func() {
|
|
err := CollectArtifacts(ctx, bootstrapClusterProxy.GetKubeconfigPath(), filepath.Join(artifactFolder, bootstrapClusterProxy.GetName(), clusterName+specName))
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
cleanInput := cleanupInput{
|
|
SpecName: specName,
|
|
Cluster: result.Cluster,
|
|
KubeconfigPath: result.KubeconfigPath,
|
|
ClusterProxy: bootstrapClusterProxy,
|
|
Namespace: namespace,
|
|
CancelWatches: cancelWatches,
|
|
IntervalsGetter: e2eConfig.GetIntervals,
|
|
SkipCleanup: skipCleanup,
|
|
ArtifactFolder: artifactFolder,
|
|
AdditionalCleanup: cleanupInstallation(ctx, clusterctlLogFolder, clusterctlConfigPath, bootstrapClusterProxy),
|
|
ClusterctlConfigPath: clusterctlConfigPath,
|
|
}
|
|
|
|
dumpSpecResourcesAndCleanup(ctx, cleanInput)
|
|
})
|
|
|
|
Context("Creating a single control-plane cluster", func() {
|
|
It("Should create a cluster with 1 worker node and can be scaled", func() {
|
|
By("Initializes with 1 worker node")
|
|
ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
|
|
ClusterProxy: bootstrapClusterProxy,
|
|
ConfigCluster: clusterctl.ConfigClusterInput{
|
|
LogFolder: clusterctlLogFolder,
|
|
ClusterctlConfigPath: clusterctlConfigPath,
|
|
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
|
|
InfrastructureProvider: "docker",
|
|
Flavor: "docker",
|
|
Namespace: namespace.Name,
|
|
ClusterName: clusterName,
|
|
KubernetesVersion: e2eConfig.MustGetVariable(KubernetesVersion),
|
|
ControlPlaneMachineCount: ptr.To(int64(1)),
|
|
WorkerMachineCount: ptr.To(int64(1)),
|
|
},
|
|
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
|
|
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
|
|
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
|
|
}, result)
|
|
|
|
By("Scaling up worker nodes to 3")
|
|
|
|
ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
|
|
ClusterProxy: bootstrapClusterProxy,
|
|
ConfigCluster: clusterctl.ConfigClusterInput{
|
|
LogFolder: clusterctlLogFolder,
|
|
ClusterctlConfigPath: clusterctlConfigPath,
|
|
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
|
|
InfrastructureProvider: "docker",
|
|
Flavor: "docker",
|
|
Namespace: namespace.Name,
|
|
ClusterName: clusterName,
|
|
KubernetesVersion: e2eConfig.MustGetVariable(KubernetesVersion),
|
|
ControlPlaneMachineCount: ptr.To(int64(1)),
|
|
WorkerMachineCount: ptr.To(int64(3)),
|
|
},
|
|
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
|
|
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
|
|
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
|
|
}, result)
|
|
|
|
By("Upgrading control plane and worker machines")
|
|
ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
|
|
ClusterProxy: bootstrapClusterProxy,
|
|
ConfigCluster: clusterctl.ConfigClusterInput{
|
|
LogFolder: clusterctlLogFolder,
|
|
ClusterctlConfigPath: clusterctlConfigPath,
|
|
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
|
|
InfrastructureProvider: "docker",
|
|
Flavor: "docker",
|
|
Namespace: namespace.Name,
|
|
ClusterName: clusterName,
|
|
KubernetesVersion: e2eConfig.MustGetVariable(KubernetesVersionUpgradeTo),
|
|
ControlPlaneMachineCount: ptr.To(int64(1)),
|
|
WorkerMachineCount: ptr.To(int64(3)),
|
|
},
|
|
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
|
|
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
|
|
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
|
|
}, result)
|
|
|
|
WaitForClusterToUpgrade(ctx, WaitForClusterToUpgradeInput{
|
|
Reader: bootstrapClusterProxy.GetClient(),
|
|
ControlPlane: result.ControlPlane,
|
|
MachineDeployments: result.MachineDeployments,
|
|
VersionAfterUpgrade: e2eConfig.MustGetVariable(KubernetesVersionUpgradeTo),
|
|
}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
|
|
|
|
WaitForControlPlaneToBeReady(ctx, WaitForControlPlaneToBeReadyInput{
|
|
Getter: bootstrapClusterProxy.GetClient(),
|
|
ControlPlane: client.ObjectKeyFromObject(result.ControlPlane),
|
|
}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
|
|
|
|
By("Scaling up control planes to 3")
|
|
|
|
ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
|
|
ClusterProxy: bootstrapClusterProxy,
|
|
ConfigCluster: clusterctl.ConfigClusterInput{
|
|
LogFolder: clusterctlLogFolder,
|
|
ClusterctlConfigPath: clusterctlConfigPath,
|
|
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
|
|
InfrastructureProvider: "docker",
|
|
Flavor: "docker",
|
|
Namespace: namespace.Name,
|
|
ClusterName: clusterName,
|
|
KubernetesVersion: e2eConfig.MustGetVariable(KubernetesVersionUpgradeTo),
|
|
ControlPlaneMachineCount: ptr.To(int64(3)),
|
|
WorkerMachineCount: ptr.To(int64(3)),
|
|
},
|
|
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
|
|
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
|
|
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
|
|
}, result)
|
|
|
|
WaitForControlPlaneToBeReady(ctx, WaitForControlPlaneToBeReadyInput{
|
|
Getter: bootstrapClusterProxy.GetClient(),
|
|
ControlPlane: client.ObjectKeyFromObject(result.ControlPlane),
|
|
}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
|
|
|
|
By("Scaling control plane nodes to 1")
|
|
|
|
ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
|
|
ClusterProxy: bootstrapClusterProxy,
|
|
ConfigCluster: clusterctl.ConfigClusterInput{
|
|
LogFolder: clusterctlLogFolder,
|
|
ClusterctlConfigPath: clusterctlConfigPath,
|
|
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
|
|
InfrastructureProvider: "docker",
|
|
Flavor: "docker",
|
|
Namespace: namespace.Name,
|
|
ClusterName: clusterName,
|
|
KubernetesVersion: e2eConfig.MustGetVariable(KubernetesVersionUpgradeTo),
|
|
ControlPlaneMachineCount: ptr.To(int64(1)),
|
|
WorkerMachineCount: ptr.To(int64(3)),
|
|
},
|
|
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
|
|
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
|
|
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
|
|
}, result)
|
|
|
|
WaitForControlPlaneToBeReady(ctx, WaitForControlPlaneToBeReadyInput{
|
|
Getter: bootstrapClusterProxy.GetClient(),
|
|
ControlPlane: client.ObjectKeyFromObject(result.ControlPlane),
|
|
}, e2eConfig.GetIntervals(specName, "wait-control-plane")...)
|
|
|
|
By("Scaling worker nodes to 1")
|
|
|
|
ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
|
|
ClusterProxy: bootstrapClusterProxy,
|
|
ConfigCluster: clusterctl.ConfigClusterInput{
|
|
LogFolder: clusterctlLogFolder,
|
|
ClusterctlConfigPath: clusterctlConfigPath,
|
|
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
|
|
InfrastructureProvider: "docker",
|
|
Flavor: "docker",
|
|
Namespace: namespace.Name,
|
|
ClusterName: clusterName,
|
|
KubernetesVersion: e2eConfig.MustGetVariable(KubernetesVersionUpgradeTo),
|
|
ControlPlaneMachineCount: ptr.To(int64(1)),
|
|
WorkerMachineCount: ptr.To(int64(1)),
|
|
},
|
|
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
|
|
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
|
|
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
|
|
}, result)
|
|
|
|
By("Checking if the cloud-init config contentFrom a secret and configmap has been applied")
|
|
machineList := GetMachinesByCluster(ctx, GetMachinesByClusterInput{
|
|
Lister: bootstrapClusterProxy.GetClient(),
|
|
ClusterName: result.Cluster.Name,
|
|
Namespace: result.Cluster.Namespace,
|
|
})
|
|
Expect(machineList.Items).ToNot(BeEmpty())
|
|
for _, machine := range machineList.Items {
|
|
if _, exists := machine.Labels["cluster.x-k8s.io/control-plane"]; exists {
|
|
bootstrapSecret := &corev1.Secret{}
|
|
Expect(bootstrapClusterProxy.GetClient().Get(ctx, types.NamespacedName{Namespace: namespace.Name, Name: *machine.Spec.Bootstrap.DataSecretName}, bootstrapSecret)).Should(Succeed())
|
|
content, ok := bootstrapSecret.Data["value"]
|
|
Expect(ok).To(BeTrue())
|
|
|
|
// define local structures to make YAML unmarshalling easy
|
|
type writeFile struct {
|
|
Path string `yaml:"path"`
|
|
Owner string `yaml:"owner"`
|
|
Permissions string `yaml:"permissions"`
|
|
Content string `yaml:"content"`
|
|
}
|
|
|
|
type cloudConfig struct {
|
|
RunCmd []string `yaml:"runcmd"`
|
|
WriteFiles []writeFile `yaml:"write_files"`
|
|
}
|
|
|
|
var cloudConfigData cloudConfig
|
|
err := yaml.Unmarshal(content, &cloudConfigData)
|
|
Expect(err).To(BeNil())
|
|
|
|
var pathContentMap = map[string]string{}
|
|
for _, file := range cloudConfigData.WriteFiles {
|
|
pathContentMap[file.Path] = file.Content
|
|
}
|
|
|
|
// Check content from all the source exists
|
|
for key, value := range map[string]string{
|
|
"/cloud-init-config-data-secret.yaml": "This content is from a secret.",
|
|
"/cloud-init-config-data-cm.yaml": "This content is from a configmap.",
|
|
"/control-plane-file.sentinel": "Just a dummy file",
|
|
} {
|
|
Expect(pathContentMap[key]).To(ContainSubstring(value))
|
|
}
|
|
}
|
|
}
|
|
})
|
|
})
|
|
})
|