463 lines
18 KiB
Go
463 lines
18 KiB
Go
/*
|
|
Copyright 2024 The Karmada Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package apiserver
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
fakeclientset "k8s.io/client-go/kubernetes/fake"
|
|
coretesting "k8s.io/client-go/testing"
|
|
"k8s.io/utils/ptr"
|
|
|
|
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
|
|
"github.com/karmada-io/karmada/operator/pkg/constants"
|
|
"github.com/karmada-io/karmada/operator/pkg/util"
|
|
)
|
|
|
|
func TestEnsureKarmadaAPIServer(t *testing.T) {
|
|
var replicas int32 = 3
|
|
image := "karmada-apiserver-image"
|
|
imagePullPolicy := corev1.PullIfNotPresent
|
|
annotations := map[string]string{"annotationKey": "annotationValue"}
|
|
labels := map[string]string{"labelKey": "labelValue"}
|
|
name := "karmada-apiserver"
|
|
namespace := "test-namespace"
|
|
serviceSubnet := "10.96.0.0/12"
|
|
|
|
cfg := &operatorv1alpha1.KarmadaComponents{
|
|
KarmadaAPIServer: &operatorv1alpha1.KarmadaAPIServer{
|
|
CommonSettings: operatorv1alpha1.CommonSettings{
|
|
Image: operatorv1alpha1.Image{ImageTag: image},
|
|
Replicas: ptr.To[int32](replicas),
|
|
Annotations: annotations,
|
|
Labels: labels,
|
|
Resources: corev1.ResourceRequirements{},
|
|
ImagePullPolicy: imagePullPolicy,
|
|
},
|
|
ServiceSubnet: ptr.To(serviceSubnet),
|
|
ExtraArgs: map[string]string{"cmd1": "arg1", "cmd2": "arg2"},
|
|
},
|
|
Etcd: &operatorv1alpha1.Etcd{
|
|
Local: &operatorv1alpha1.LocalEtcd{},
|
|
},
|
|
}
|
|
|
|
fakeClient := fakeclientset.NewSimpleClientset()
|
|
|
|
err := EnsureKarmadaAPIServer(fakeClient, cfg, name, namespace, map[string]bool{})
|
|
if err != nil {
|
|
t.Fatalf("expected no error, but got: %v", err)
|
|
}
|
|
|
|
actions := fakeClient.Actions()
|
|
if len(actions) != 2 {
|
|
t.Fatalf("expected 2 actions, but got %d", len(actions))
|
|
}
|
|
}
|
|
|
|
func TestEnsureKarmadaAggregatedAPIServer(t *testing.T) {
|
|
var replicas int32 = 2
|
|
image := "karmada-aggregated-apiserver-image"
|
|
imagePullPolicy := corev1.PullIfNotPresent
|
|
annotationValues := map[string]string{"annotationKey": "annotationValue"}
|
|
labelValues := map[string]string{"labelKey": "labelValue"}
|
|
name := "test-agg-server"
|
|
namespace := "test-namespace"
|
|
|
|
cfg := &operatorv1alpha1.KarmadaComponents{
|
|
KarmadaAggregatedAPIServer: &operatorv1alpha1.KarmadaAggregatedAPIServer{
|
|
CommonSettings: operatorv1alpha1.CommonSettings{
|
|
Image: operatorv1alpha1.Image{ImageTag: image},
|
|
Replicas: ptr.To[int32](replicas),
|
|
Annotations: annotationValues,
|
|
Labels: labelValues,
|
|
Resources: corev1.ResourceRequirements{},
|
|
ImagePullPolicy: imagePullPolicy,
|
|
},
|
|
ExtraArgs: map[string]string{"cmd1": "arg1", "cmd2": "arg2"},
|
|
},
|
|
Etcd: &operatorv1alpha1.Etcd{
|
|
Local: &operatorv1alpha1.LocalEtcd{},
|
|
},
|
|
}
|
|
|
|
featureGates := map[string]bool{"FeatureA": true}
|
|
|
|
fakeClient := fakeclientset.NewSimpleClientset()
|
|
|
|
err := EnsureKarmadaAggregatedAPIServer(fakeClient, cfg, name, namespace, featureGates)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, but got: %v", err)
|
|
}
|
|
|
|
actions := fakeClient.Actions()
|
|
if len(actions) != 2 {
|
|
t.Fatalf("expected 2 actions, but got %d", len(actions))
|
|
}
|
|
}
|
|
|
|
func TestInstallKarmadaAPIServer(t *testing.T) {
|
|
var replicas int32 = 3
|
|
image := "karmada-apiserver-image"
|
|
imagePullPolicy := corev1.PullIfNotPresent
|
|
annotations := map[string]string{"annotationKey": "annotationValue"}
|
|
labels := map[string]string{"labelKey": "labelValue"}
|
|
name := "karmada-apiserver"
|
|
namespace := "test-namespace"
|
|
serviceSubnet := "10.96.0.0/12"
|
|
priorityClassName := "system-cluster-critical"
|
|
|
|
// Create fake clientset.
|
|
fakeClient := fakeclientset.NewSimpleClientset()
|
|
|
|
// Define a valid KarmadaAPIServer configuration.
|
|
cfg := &operatorv1alpha1.KarmadaAPIServer{
|
|
CommonSettings: operatorv1alpha1.CommonSettings{
|
|
Image: operatorv1alpha1.Image{ImageTag: image},
|
|
Replicas: ptr.To[int32](replicas),
|
|
Annotations: annotations,
|
|
Labels: labels,
|
|
Resources: corev1.ResourceRequirements{},
|
|
ImagePullPolicy: imagePullPolicy,
|
|
PriorityClassName: priorityClassName,
|
|
},
|
|
ServiceSubnet: ptr.To(serviceSubnet),
|
|
ExtraArgs: map[string]string{"cmd1": "arg1", "cmd2": "arg2"},
|
|
}
|
|
etcdCfg := &operatorv1alpha1.Etcd{
|
|
Local: &operatorv1alpha1.LocalEtcd{},
|
|
}
|
|
featureGates := map[string]bool{"FeatureA": true}
|
|
|
|
// Call the function under test.
|
|
err := installKarmadaAPIServer(fakeClient, cfg, etcdCfg, name, namespace, featureGates)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, but got: %v", err)
|
|
}
|
|
|
|
deployment, err := verifyDeploymentCreation(fakeClient, &replicas, imagePullPolicy, cfg.ExtraArgs, name, namespace, image, util.KarmadaAPIServerName(name), priorityClassName)
|
|
if err != nil {
|
|
t.Fatalf("failed to verify karmada apiserver correct deployment creation correct details: %v", err)
|
|
}
|
|
|
|
err = verifyAPIServerDeploymentAdditionalDetails(deployment, name, serviceSubnet)
|
|
if err != nil {
|
|
t.Errorf("failed to verify karmada apiserver additional deployment details: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCreateKarmadaAPIServerService(t *testing.T) {
|
|
// Initialize fake clientset.
|
|
client := fakeclientset.NewSimpleClientset()
|
|
|
|
// Define inputs.
|
|
name := "test-apiserver"
|
|
namespace := "test-namespace"
|
|
cfg := &operatorv1alpha1.KarmadaAPIServer{
|
|
ServiceAnnotations: map[string]string{"annotationKey": "annotationValue"},
|
|
}
|
|
|
|
// Call the function under test.
|
|
err := createKarmadaAPIServerService(client, cfg, name, namespace)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, but got: %v", err)
|
|
}
|
|
|
|
// Ensure the expected action (service creation) occurred.
|
|
actions := client.Actions()
|
|
if len(actions) != 1 {
|
|
t.Fatalf("expected 1 action, but got %d actions", len(actions))
|
|
}
|
|
|
|
// Validate the action is a CreateAction and it's for the correct resource (Service).
|
|
createAction, ok := actions[0].(coretesting.CreateAction)
|
|
if !ok {
|
|
t.Fatalf("expected CreateAction, but got %T", actions[0])
|
|
}
|
|
|
|
if createAction.GetResource().Resource != "services" {
|
|
t.Fatalf("expected action on 'services', but got '%s'", createAction.GetResource().Resource)
|
|
}
|
|
|
|
// Validate the created service object.
|
|
service := createAction.GetObject().(*corev1.Service)
|
|
expectedServiceName := util.KarmadaAPIServerName(name)
|
|
if service.Name != expectedServiceName {
|
|
t.Fatalf("expected service name '%s', but got '%s'", expectedServiceName, service.Name)
|
|
}
|
|
|
|
if service.Namespace != namespace {
|
|
t.Fatalf("expected service namespace '%s', but got '%s'", namespace, service.Namespace)
|
|
}
|
|
|
|
if _, exists := service.Annotations["annotationKey"]; !exists {
|
|
t.Errorf("expected annotation with key 'annotationKey' and value 'annotationValue', but it was missing")
|
|
}
|
|
}
|
|
|
|
func TestInstallKarmadaAggregatedAPIServer(t *testing.T) {
|
|
var replicas int32 = 2
|
|
image := "karmada-aggregated-apiserver-image"
|
|
imagePullPolicy := corev1.PullIfNotPresent
|
|
annotationValues := map[string]string{"annotationKey": "annotationValue"}
|
|
labelValues := map[string]string{"labelKey": "labelValue"}
|
|
name := "test-agg-server"
|
|
namespace := "test-namespace"
|
|
priorityClassName := "system-cluster-critical"
|
|
|
|
// Use fake clientset.
|
|
fakeClient := fakeclientset.NewSimpleClientset()
|
|
|
|
// Define valid inputs.
|
|
cfg := &operatorv1alpha1.KarmadaAggregatedAPIServer{
|
|
CommonSettings: operatorv1alpha1.CommonSettings{
|
|
Image: operatorv1alpha1.Image{ImageTag: image},
|
|
Replicas: ptr.To[int32](replicas),
|
|
Annotations: annotationValues,
|
|
Labels: labelValues,
|
|
Resources: corev1.ResourceRequirements{},
|
|
ImagePullPolicy: imagePullPolicy,
|
|
PriorityClassName: priorityClassName,
|
|
},
|
|
ExtraArgs: map[string]string{"cmd1": "arg1", "cmd2": "arg2"},
|
|
}
|
|
|
|
featureGates := map[string]bool{"FeatureA": true}
|
|
etcdCfg := &operatorv1alpha1.Etcd{
|
|
Local: &operatorv1alpha1.LocalEtcd{},
|
|
}
|
|
err := installKarmadaAggregatedAPIServer(fakeClient, cfg, etcdCfg, name, namespace, featureGates)
|
|
if err != nil {
|
|
t.Fatalf("Failed to install Karmada Aggregated API Server: %v", err)
|
|
}
|
|
|
|
deployment, err := verifyDeploymentCreation(fakeClient, &replicas, imagePullPolicy, cfg.ExtraArgs, name, namespace, image, util.KarmadaAggregatedAPIServerName(name), priorityClassName)
|
|
if err != nil {
|
|
t.Fatalf("failed to verify karmada aggregated apiserver deployment creation correct details: %v", err)
|
|
}
|
|
|
|
err = verifyAggregatedAPIServerDeploymentAdditionalDetails(featureGates, deployment, name)
|
|
if err != nil {
|
|
t.Errorf("failed to verify karmada aggregated apiserver additional deployment details: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCreateKarmadaAggregatedAPIServerService(t *testing.T) {
|
|
// Initialize fake clientset.
|
|
client := fakeclientset.NewSimpleClientset()
|
|
|
|
// Define inputs.
|
|
name := "test-agg-server"
|
|
namespace := "test-namespace"
|
|
|
|
// Call the function under test.
|
|
err := createKarmadaAggregatedAPIServerService(client, name, namespace)
|
|
if err != nil {
|
|
t.Fatalf("expected no error, but got: %v", err)
|
|
}
|
|
|
|
// Ensure the expected action (service creation) occurred.
|
|
actions := client.Actions()
|
|
if len(actions) != 1 {
|
|
t.Fatalf("expected 1 action, but got %d actions", len(actions))
|
|
}
|
|
|
|
// Validate the action is a CreateAction and it's for the correct resource (Service).
|
|
createAction, ok := actions[0].(coretesting.CreateAction)
|
|
if !ok {
|
|
t.Fatalf("expected CreateAction, but got %T", actions[0])
|
|
}
|
|
|
|
if createAction.GetResource().Resource != "services" {
|
|
t.Fatalf("expected action on 'services', but got '%s'", createAction.GetResource().Resource)
|
|
}
|
|
|
|
// Validate the created service object.
|
|
service := createAction.GetObject().(*corev1.Service)
|
|
expectedServiceName := util.KarmadaAggregatedAPIServerName(name)
|
|
if service.Name != expectedServiceName {
|
|
t.Fatalf("expected service name '%s', but got '%s'", expectedServiceName, service.Name)
|
|
}
|
|
|
|
if service.Namespace != namespace {
|
|
t.Fatalf("expected service namespace '%s', but got '%s'", namespace, service.Namespace)
|
|
}
|
|
}
|
|
|
|
// contains check if a slice contains a specific string.
|
|
func contains(slice []string, item string) bool {
|
|
for _, s := range slice {
|
|
if s == item {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// verifyDeploymentCreation verifies the creation of a Kubernetes deployment
|
|
// based on the given parameters. It ensures that the deployment has the correct
|
|
// number of replicas, image pull policy, extra arguments, and labels, as well
|
|
// as the correct image for the Karmada API server.
|
|
func verifyDeploymentCreation(client *fakeclientset.Clientset, replicas *int32, imagePullPolicy corev1.PullPolicy, extraArgs map[string]string, name, namespace, image, expectedDeploymentName, priorityClassName string) (*appsv1.Deployment, error) {
|
|
// Assert that a Deployment was created.
|
|
actions := client.Actions()
|
|
if len(actions) != 1 {
|
|
return nil, fmt.Errorf("expected exactly 1 action either create or update, but got %d actions", len(actions))
|
|
}
|
|
|
|
// Check that the action was a Deployment creation.
|
|
createAction, ok := actions[0].(coretesting.CreateAction)
|
|
if !ok {
|
|
return nil, fmt.Errorf("expected a CreateAction, but got %T", actions[0])
|
|
}
|
|
|
|
// Check that the action was performed on the correct resource.
|
|
if createAction.GetResource().Resource != "deployments" {
|
|
return nil, fmt.Errorf("expected action on 'deployments', but got '%s'", createAction.GetResource().Resource)
|
|
}
|
|
|
|
deployment := createAction.GetObject().(*appsv1.Deployment)
|
|
err := verifyDeploymentDetails(deployment, replicas, imagePullPolicy, extraArgs, name, namespace, image, expectedDeploymentName, priorityClassName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return deployment, nil
|
|
}
|
|
|
|
// verifyDeploymentDetails ensures that the specified deployment contains the
|
|
// correct configuration for replicas, image pull policy, extra args, and image.
|
|
// It validates that the deployment matches the expected Karmada API server settings.
|
|
func verifyDeploymentDetails(deployment *appsv1.Deployment, replicas *int32, imagePullPolicy corev1.PullPolicy, extraArgs map[string]string, name, namespace, image, expectedDeploymentName, priorityClassName string) error {
|
|
if deployment.Name != expectedDeploymentName {
|
|
return fmt.Errorf("expected deployment name '%s', but got '%s'", expectedDeploymentName, deployment.Name)
|
|
}
|
|
|
|
if deployment.Spec.Template.Spec.PriorityClassName != priorityClassName {
|
|
return fmt.Errorf("expected priorityClassName to be set to %s, but got %s", priorityClassName, deployment.Spec.Template.Spec.PriorityClassName)
|
|
}
|
|
|
|
expectedNamespace := "test-namespace"
|
|
if deployment.Namespace != expectedNamespace {
|
|
return fmt.Errorf("expected deployment namespace '%s', but got '%s'", expectedNamespace, deployment.Namespace)
|
|
}
|
|
|
|
if _, exists := deployment.Annotations["annotationKey"]; !exists {
|
|
return fmt.Errorf("expected annotation with key 'annotationKey' and value 'annotationValue', but it was missing")
|
|
}
|
|
|
|
if _, exists := deployment.Labels["labelKey"]; !exists {
|
|
return fmt.Errorf("expected label with key 'labelKey' and value 'labelValue', but it was missing")
|
|
}
|
|
|
|
if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != *replicas {
|
|
return fmt.Errorf("expected replicas to be %d, but got %d", replicas, *deployment.Spec.Replicas)
|
|
}
|
|
|
|
containers := deployment.Spec.Template.Spec.Containers
|
|
if len(containers) != 1 {
|
|
return fmt.Errorf("expected exactly 1 container, but got %d", len(containers))
|
|
}
|
|
|
|
expectedImage := fmt.Sprintf(":%s", image)
|
|
container := containers[0]
|
|
if container.Image != expectedImage {
|
|
return fmt.Errorf("expected container image '%s', but got '%s'", expectedImage, container.Image)
|
|
}
|
|
|
|
if container.ImagePullPolicy != imagePullPolicy {
|
|
return fmt.Errorf("expected image pull policy '%s', but got '%s'", imagePullPolicy, container.ImagePullPolicy)
|
|
}
|
|
|
|
for key, value := range extraArgs {
|
|
expectedArg := fmt.Sprintf("--%s=%s", key, value)
|
|
if !contains(container.Command, expectedArg) {
|
|
return fmt.Errorf("expected container commands to include '%s', but it was missing", expectedArg)
|
|
}
|
|
}
|
|
|
|
etcdServersArg := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d,", util.KarmadaEtcdClientName(name), namespace, constants.EtcdListenClientPort)
|
|
etcdServersArg = fmt.Sprintf("--etcd-servers=%s", etcdServersArg[:len(etcdServersArg)-1])
|
|
if !contains(container.Command, etcdServersArg) {
|
|
return fmt.Errorf("etcd servers argument '%s' not found in container command", etcdServersArg)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// verifyAggregatedAPIServerDeploymentAdditionalDetails validates the additional
|
|
// details of the Karmada Aggregated API server deployment, such as the service
|
|
// subnet and configuration related to aggregated API server behavior.
|
|
func verifyAggregatedAPIServerDeploymentAdditionalDetails(featureGates map[string]bool, deployment *appsv1.Deployment, expectedDeploymentName string) error {
|
|
var featureGatesArg string
|
|
for key, value := range featureGates {
|
|
featureGatesArg += fmt.Sprintf("%s=%t,", key, value)
|
|
}
|
|
featureGatesArg = fmt.Sprintf("--feature-gates=%s", featureGatesArg[:len(featureGatesArg)-1])
|
|
if !contains(deployment.Spec.Template.Spec.Containers[0].Command, featureGatesArg) {
|
|
return fmt.Errorf("expected container commands to include '%s', but it was missing", featureGatesArg)
|
|
}
|
|
|
|
if len(deployment.Spec.Template.Spec.Volumes) != 3 {
|
|
return fmt.Errorf("expected 3 volumes, but found %d", len(deployment.Spec.Template.Spec.Volumes))
|
|
}
|
|
|
|
var extractedSecrets []string
|
|
for _, volume := range deployment.Spec.Template.Spec.Volumes {
|
|
extractedSecrets = append(extractedSecrets, volume.Secret.SecretName)
|
|
}
|
|
expectedSecrets := []string{util.ComponentKarmadaConfigSecretName(util.KarmadaAggregatedAPIServerName(expectedDeploymentName)), util.KarmadaCertSecretName(expectedDeploymentName), util.EtcdCertSecretName(expectedDeploymentName)}
|
|
for _, expectedSecret := range expectedSecrets {
|
|
if !contains(extractedSecrets, expectedSecret) {
|
|
return fmt.Errorf("expected secret '%s' not found in extracted secrets", expectedSecret)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// verifyAPIServerDeploymentAdditionalDetails checks the additional configuration
|
|
// details of a Kubernetes deployment for the Karmada API server. It validates the
|
|
// service cluster IP range, the number of volumes, and ensures that the required
|
|
// secret volumes are mounted in the deployment.
|
|
func verifyAPIServerDeploymentAdditionalDetails(deployment *appsv1.Deployment, expectedDeploymentName, serviceSubnet string) error {
|
|
serviceClusterIPRangeArg := fmt.Sprintf("--service-cluster-ip-range=%s", serviceSubnet)
|
|
if !contains(deployment.Spec.Template.Spec.Containers[0].Command, serviceClusterIPRangeArg) {
|
|
return fmt.Errorf("service cluster IP range argument '%s' not found in container command", serviceClusterIPRangeArg)
|
|
}
|
|
|
|
if len(deployment.Spec.Template.Spec.Volumes) != 2 {
|
|
return fmt.Errorf("expected 2 volumes, but found %d", len(deployment.Spec.Template.Spec.Volumes))
|
|
}
|
|
|
|
var extractedSecrets []string
|
|
for _, volume := range deployment.Spec.Template.Spec.Volumes {
|
|
extractedSecrets = append(extractedSecrets, volume.Secret.SecretName)
|
|
}
|
|
expectedSecrets := []string{util.KarmadaCertSecretName(expectedDeploymentName), util.EtcdCertSecretName(expectedDeploymentName)}
|
|
for _, expectedSecret := range expectedSecrets {
|
|
if !contains(extractedSecrets, expectedSecret) {
|
|
return fmt.Errorf("expected secret '%s' not found in extracted secrets", expectedSecret)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|