437 lines
16 KiB
Go
437 lines
16 KiB
Go
package controllers
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
// "strconv"
|
|
|
|
kubefloworgv1alpha1 "github.com/kubeflow/kubeflow/components/pvc-viewer/api/v1alpha1"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
)
|
|
|
|
var (
|
|
// We're using these variables to keep track of the test-# and create a unique namespace for each test
|
|
testCount = 0
|
|
testHelper *TestHelper
|
|
)
|
|
|
|
// +kubebuilder:docs-gen:collapse=Imports
|
|
|
|
var _ = Describe("PVCViewer controller", func() {
|
|
|
|
// BeforeEach provides a clean namespace for each test
|
|
BeforeEach(func() {
|
|
// Each test should run in isolation. Using Namespaces is a good way to do this.
|
|
// However, while EnvTest supports creating namespaces, it can't tear them down.
|
|
// It is recommended to simply use a different namespace for each test.
|
|
// See: https://book.kubebuilder.io/reference/envtest.html#namespace-usage-limitation
|
|
testCount++
|
|
testHelper = &TestHelper{
|
|
namespace: "test-" + fmt.Sprint(testCount),
|
|
ctx: ctx,
|
|
k8sClient: k8sClient,
|
|
}
|
|
|
|
ns := &corev1.Namespace{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: testHelper.namespace,
|
|
},
|
|
}
|
|
Expect(k8sClient.Create(ctx, ns)).Should(Succeed())
|
|
})
|
|
|
|
AfterEach(func() {
|
|
// Delete objects in test namespace
|
|
// Lets tests run deterministically, speeds them up and increases debugability
|
|
objectsToDelete := []client.Object{
|
|
&kubefloworgv1alpha1.PVCViewer{},
|
|
&appsv1.Deployment{},
|
|
&corev1.Service{},
|
|
&corev1.Pod{},
|
|
virtualServiceTemplate.DeepCopy(),
|
|
}
|
|
for _, object := range objectsToDelete {
|
|
Expect(k8sClient.DeleteAllOf(ctx, object, client.InNamespace(testHelper.namespace))).Should(Succeed())
|
|
}
|
|
})
|
|
|
|
// Test the validation and defaulting webhooks
|
|
Context("Defaulting and Validating Webhooks", func() {
|
|
It("Should create a PodSpec", func() {
|
|
pvcViewer := testHelper.CreateViewer(&kubefloworgv1alpha1.PVCViewerSpec{
|
|
PVC: "test-pvc",
|
|
})
|
|
|
|
Expect(pvcViewer.Spec.PodSpec).ShouldNot(BeNil())
|
|
Expect(pvcViewer.Spec.PodSpec.Containers).Should(HaveLen(1))
|
|
Expect(pvcViewer.Spec.PodSpec.Containers[0].Image).ShouldNot(HaveLen(0))
|
|
|
|
Expect(pvcViewer.Spec.PodSpec.Volumes).Should(HaveLen(1))
|
|
Expect(pvcViewer.Spec.PodSpec.Volumes[0].PersistentVolumeClaim.ClaimName).Should(Equal("test-pvc"))
|
|
})
|
|
|
|
It("Should use defaults if environment variable is set", func() {
|
|
filePath, _ := filepath.Abs("testdata/podspec_default.yaml")
|
|
os.Setenv(kubefloworgv1alpha1.DefaultPodSpecPathEnvName, filePath)
|
|
defer os.Unsetenv(kubefloworgv1alpha1.DefaultPodSpecPathEnvName)
|
|
|
|
pvcViewer := testHelper.CreateViewer(&kubefloworgv1alpha1.PVCViewerSpec{
|
|
PVC: "test-pvc",
|
|
})
|
|
|
|
Expect(pvcViewer.Spec.PodSpec).ShouldNot(BeNil())
|
|
Expect(pvcViewer.Spec.PodSpec.Containers).Should(HaveLen(1))
|
|
Expect(pvcViewer.Spec.PodSpec.Containers[0].Name).Should(Equal("test"))
|
|
Expect(pvcViewer.Spec.PodSpec.SecurityContext).ShouldNot(BeNil())
|
|
Expect(*pvcViewer.Spec.PodSpec.SecurityContext.RunAsUser).Should(Equal(int64(1234)))
|
|
})
|
|
|
|
It("The spec.PVC must be specified", func() {
|
|
pvcViewer := &kubefloworgv1alpha1.PVCViewer{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-pvcviewer",
|
|
Namespace: testHelper.namespace,
|
|
},
|
|
Spec: kubefloworgv1alpha1.PVCViewerSpec{
|
|
PVC: "",
|
|
},
|
|
}
|
|
err := k8sClient.Create(ctx, pvcViewer)
|
|
Expect(err).Should(HaveOccurred())
|
|
Expect(err.Error()).Should(ContainSubstring("denied the request: PVC name must be specified"))
|
|
})
|
|
|
|
It("Not using the spec.PVC in podSpec.volumes is forbidden", func() {
|
|
pvcViewer := &kubefloworgv1alpha1.PVCViewer{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-pvcviewer",
|
|
Namespace: testHelper.namespace,
|
|
},
|
|
Spec: kubefloworgv1alpha1.PVCViewerSpec{
|
|
PVC: "test-pvc",
|
|
PodSpec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "test",
|
|
Image: "test",
|
|
},
|
|
},
|
|
Volumes: []corev1.Volume{},
|
|
},
|
|
},
|
|
}
|
|
err := k8sClient.Create(ctx, pvcViewer)
|
|
Expect(err).Should(HaveOccurred())
|
|
Expect(err.Error()).Should(ContainSubstring("denied the request: PVC test-pvc must be used in the podSpec"))
|
|
})
|
|
})
|
|
|
|
// Test simple CRUD operations, i.e. the creation and update of deployments, services, virtualservices, etc.
|
|
Context("When PVCViewer created", func() {
|
|
|
|
It("Should CRUD Deployment", func() {
|
|
By("By creating a viewer")
|
|
pvcViewer := testHelper.CreateViewer(&kubefloworgv1alpha1.PVCViewerSpec{
|
|
PVC: "test-pvc",
|
|
})
|
|
|
|
deployment := &appsv1.Deployment{}
|
|
Eventually(func() error {
|
|
return testHelper.GetRelatedResource(pvcViewer, deployment)
|
|
}, timeout, interval).Should(Succeed())
|
|
Expect(deployment).ShouldNot(BeNil())
|
|
Expect(deployment.Spec.Template.Spec.Containers).Should(HaveLen(1))
|
|
Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(pvcViewer.Spec.PodSpec.Containers[0].Image))
|
|
Expect(deployment.ObjectMeta.Labels).Should(And(
|
|
HaveKeyWithValue(nameLabelKey, pvcViewer.Name),
|
|
HaveKeyWithValue(instanceLabelKey, resourcePrefix+pvcViewer.Name),
|
|
HaveKeyWithValue(partOfLabelKey, partOfLabelValue)))
|
|
|
|
By("Updating the PVCViewer")
|
|
newImageName := "sometestimage:123"
|
|
newContainerName := "test-container"
|
|
Eventually(func() error {
|
|
viewer := &kubefloworgv1alpha1.PVCViewer{}
|
|
if err := testHelper.GetRelatedResource(pvcViewer, viewer); err != nil {
|
|
return err
|
|
}
|
|
viewer.Spec.PodSpec.Containers[0].Image = newImageName
|
|
viewer.Spec.PodSpec.Containers = append(viewer.Spec.PodSpec.Containers,
|
|
corev1.Container{
|
|
Name: newContainerName,
|
|
Image: newImageName,
|
|
})
|
|
return k8sClient.Update(ctx, viewer)
|
|
}, timeout, interval).Should(Succeed())
|
|
|
|
Eventually(func() (int, error) {
|
|
if err := testHelper.GetRelatedResource(pvcViewer, deployment); err != nil {
|
|
return -1, err
|
|
}
|
|
return len(deployment.Spec.Template.Spec.Containers), nil
|
|
}, timeout, interval).Should(Equal(2))
|
|
Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(newImageName))
|
|
Expect(deployment.Spec.Template.Spec.Containers[1].Name).Should(Equal(newContainerName))
|
|
Expect(deployment.Spec.Template.Spec.Containers[1].Image).Should(Equal(newImageName))
|
|
|
|
By("Deleting the PVCViewer")
|
|
testHelper.ExpectMatchingOwnerReferences(pvcViewer, deployment.ObjectMeta.OwnerReferences)
|
|
})
|
|
|
|
It("Should CRUD Services", func() {
|
|
By("Creating an empty Viewer, a service should not exist")
|
|
pvcViewer := testHelper.CreateViewer(&kubefloworgv1alpha1.PVCViewerSpec{
|
|
PVC: "test-pvc",
|
|
})
|
|
service := &corev1.Service{}
|
|
Consistently(func() error {
|
|
return testHelper.GetRelatedResource(pvcViewer, service)
|
|
}, duration, interval).ShouldNot(Succeed())
|
|
|
|
By("Adding a Service to the PVCViewer")
|
|
targetPort := 1234
|
|
Eventually(func() error {
|
|
viewer := &kubefloworgv1alpha1.PVCViewer{}
|
|
if err := testHelper.GetRelatedResource(pvcViewer, viewer); err != nil {
|
|
return err
|
|
}
|
|
viewer.Spec.Networking = kubefloworgv1alpha1.Networking{
|
|
TargetPort: intstr.IntOrString{IntVal: int32(targetPort)},
|
|
}
|
|
return k8sClient.Update(ctx, viewer)
|
|
}, timeout, interval).Should(Succeed())
|
|
|
|
Eventually(func() error {
|
|
return testHelper.GetRelatedResource(pvcViewer, service)
|
|
}, timeout, interval).Should(Succeed())
|
|
Expect(service).ShouldNot(BeNil())
|
|
Expect(service.Spec.Ports).Should(HaveLen(1))
|
|
Expect(service.Spec.Ports[0].Name).Should(Equal("http"))
|
|
Expect(service.Spec.Ports[0].Port).Should(Equal(servicePort))
|
|
Expect(service.Spec.Ports[0].TargetPort).Should(Equal(intstr.IntOrString{IntVal: int32(targetPort)}))
|
|
Expect(service.ObjectMeta.Labels).Should(And(
|
|
HaveKeyWithValue(nameLabelKey, pvcViewer.Name),
|
|
HaveKeyWithValue(instanceLabelKey, resourcePrefix+pvcViewer.Name),
|
|
HaveKeyWithValue(partOfLabelKey, partOfLabelValue)))
|
|
|
|
By("Updating the Service in the PVCViewer")
|
|
newTargetPort := 5678
|
|
Eventually(func() error {
|
|
viewer := &kubefloworgv1alpha1.PVCViewer{}
|
|
if err := testHelper.GetRelatedResource(pvcViewer, viewer); err != nil {
|
|
return err
|
|
}
|
|
viewer.Spec.Networking.TargetPort = intstr.IntOrString{IntVal: int32(newTargetPort)}
|
|
return k8sClient.Update(ctx, viewer)
|
|
}, timeout, interval).Should(Succeed())
|
|
|
|
Eventually(func() (bool, error) {
|
|
if err := testHelper.GetRelatedResource(pvcViewer, service); err != nil {
|
|
return false, err
|
|
}
|
|
return service.Spec.Ports[0].TargetPort == intstr.IntOrString{IntVal: int32(newTargetPort)}, nil
|
|
}, timeout, interval).Should(BeTrue())
|
|
|
|
By("Deleting the PVCViewer")
|
|
testHelper.ExpectMatchingOwnerReferences(pvcViewer, service.ObjectMeta.OwnerReferences)
|
|
})
|
|
|
|
It("Should CRUD VirtualService", func() {
|
|
By("Creating a default viewer, a VirtualService should not exist")
|
|
pvcViewer := testHelper.CreateViewer(&kubefloworgv1alpha1.PVCViewerSpec{
|
|
PVC: "test-pvc",
|
|
})
|
|
virtualService := &unstructured.Unstructured{
|
|
Object: map[string]interface{}{
|
|
"apiVersion": "networking.istio.io/v1alpha3",
|
|
"kind": "VirtualService",
|
|
},
|
|
}
|
|
Consistently(func() error {
|
|
return testHelper.GetRelatedResource(pvcViewer, virtualService)
|
|
}, duration, interval).ShouldNot(Succeed())
|
|
|
|
By("Adding a minimal VirtualService")
|
|
basePrefix := "/base"
|
|
Eventually(func() error {
|
|
viewer := &kubefloworgv1alpha1.PVCViewer{}
|
|
if err := testHelper.GetRelatedResource(pvcViewer, viewer); err != nil {
|
|
return err
|
|
}
|
|
viewer.Spec.Networking = kubefloworgv1alpha1.Networking{
|
|
TargetPort: intstr.IntOrString{IntVal: 80},
|
|
BasePrefix: basePrefix,
|
|
}
|
|
return k8sClient.Update(ctx, viewer)
|
|
}, timeout, interval).Should(Succeed())
|
|
|
|
Eventually(func() error {
|
|
return testHelper.GetRelatedResource(pvcViewer, virtualService)
|
|
}, timeout, interval).Should(Succeed())
|
|
Expect(virtualService.Object["metadata"].(map[string]interface{})["labels"]).Should(And(
|
|
HaveKeyWithValue(nameLabelKey, pvcViewer.Name),
|
|
HaveKeyWithValue(instanceLabelKey, resourcePrefix+pvcViewer.Name),
|
|
HaveKeyWithValue(partOfLabelKey, partOfLabelValue)))
|
|
http := virtualService.Object["spec"].(map[string]interface{})["http"]
|
|
Expect(http).Should(HaveLen(1))
|
|
http0 := http.([]interface{})[0].(map[string]interface{})
|
|
Expect(http0["timeout"]).Should(BeNil())
|
|
expectedRewrite := fmt.Sprintf("%s/%s/%s/", basePrefix, testHelper.namespace, pvcViewer.Name)
|
|
Expect(http0["match"].([]interface{})[0].(map[string]interface{})["uri"].(map[string]interface{})["prefix"]).Should(Equal(expectedRewrite))
|
|
Expect(http0["rewrite"].(map[string]interface{})["uri"]).Should(Equal(expectedRewrite))
|
|
|
|
// Test the status.URL gets set correctly
|
|
Eventually(func() (string, error) {
|
|
viewer := &kubefloworgv1alpha1.PVCViewer{}
|
|
if err := testHelper.GetRelatedResource(pvcViewer, viewer); err != nil || viewer.Status.URL == nil {
|
|
return "", err
|
|
}
|
|
return *viewer.Status.URL, nil
|
|
}, timeout, interval).Should(Equal(expectedRewrite))
|
|
|
|
By("Updating the defaults")
|
|
newBasePrefix := "/newbase"
|
|
newTimeout := "10s"
|
|
newRewrite := "/newrewrite"
|
|
Eventually(func() error {
|
|
viewer := &kubefloworgv1alpha1.PVCViewer{}
|
|
if err := testHelper.GetRelatedResource(pvcViewer, viewer); err != nil {
|
|
return err
|
|
}
|
|
viewer.Spec.Networking.BasePrefix = newBasePrefix
|
|
viewer.Spec.Networking.Timeout = newTimeout
|
|
viewer.Spec.Networking.Rewrite = newRewrite
|
|
return k8sClient.Update(ctx, viewer)
|
|
}, timeout, interval).Should(Succeed())
|
|
|
|
Eventually(func() (bool, error) {
|
|
if err := testHelper.GetRelatedResource(pvcViewer, virtualService); err != nil {
|
|
return false, err
|
|
}
|
|
return virtualService.Object["spec"].(map[string]interface{})["http"].([]interface{})[0].(map[string]interface{})["timeout"] != nil, nil
|
|
}, timeout, interval).Should(BeTrue())
|
|
Expect(virtualService.Object["metadata"].(map[string]interface{})["ownerReferences"]).Should(HaveLen(1))
|
|
http = virtualService.Object["spec"].(map[string]interface{})["http"]
|
|
Expect(http).Should(HaveLen(1))
|
|
http0 = http.([]interface{})[0].(map[string]interface{})
|
|
Expect(http0["timeout"]).Should(Equal(newTimeout))
|
|
expectedRewrite = fmt.Sprintf("%s/%s/%s/", newBasePrefix, testHelper.namespace, pvcViewer.Name)
|
|
Expect(http0["match"].([]interface{})[0].(map[string]interface{})["uri"].(map[string]interface{})["prefix"]).Should(Equal(expectedRewrite))
|
|
Expect(http0["rewrite"].(map[string]interface{})["uri"]).Should(Equal(newRewrite))
|
|
|
|
By("Deleting the PVCViewer")
|
|
Expect(virtualService.Object["metadata"].(map[string]interface{})["ownerReferences"]).Should(HaveLen(1))
|
|
})
|
|
|
|
It("Should update status", func() {
|
|
By("By checking a default PVCViewer's status")
|
|
pvcViewer := testHelper.CreateViewer(&kubefloworgv1alpha1.PVCViewerSpec{
|
|
PVC: "test-pvc",
|
|
})
|
|
Consistently(func() (bool, error) {
|
|
viewer := &kubefloworgv1alpha1.PVCViewer{}
|
|
if err := testHelper.GetRelatedResource(pvcViewer, viewer); err != nil {
|
|
return true, err
|
|
}
|
|
return viewer.Status.Ready, nil
|
|
}, duration, interval).Should(Equal(false))
|
|
|
|
deployment := &appsv1.Deployment{}
|
|
Eventually(func() error {
|
|
return testHelper.GetRelatedResource(pvcViewer, deployment)
|
|
}, timeout, interval).Should(Succeed())
|
|
|
|
By("By updating the Deployment's status")
|
|
// We need to update both ReadyReplicas and Replicas or else CRD validation will fail.
|
|
deployment.Status.ReadyReplicas = 1
|
|
deployment.Status.Replicas = 1
|
|
Expect(k8sClient.Status().Update(ctx, deployment)).Should(Succeed())
|
|
|
|
viewer := &kubefloworgv1alpha1.PVCViewer{}
|
|
Eventually(func() (bool, error) {
|
|
err := testHelper.GetRelatedResource(pvcViewer, viewer)
|
|
return viewer.Status.Ready, err
|
|
}, timeout, interval).Should(BeTrue())
|
|
Expect(deployment.Status.ReadyReplicas).Should(Equal(int32(1)))
|
|
})
|
|
})
|
|
|
|
// Test affinity generation for RWO scheduling
|
|
Context("When RWO Scheduling is used", func() {
|
|
It("Does not generate affinities for RWX PVCs", func() {
|
|
By("Creating a RWX PVC and mounting Pod")
|
|
pvcName := "rwx-pvc"
|
|
testHelper.CreatePVC(pvcName, corev1.ReadWriteMany)
|
|
testHelper.CreatePod("rwx-pod", "some-node", pvcName)
|
|
|
|
By("Creating a PVCViewer for the RWX PVC")
|
|
pvcViewer := testHelper.CreateViewer(&kubefloworgv1alpha1.PVCViewerSpec{
|
|
PVC: pvcName,
|
|
})
|
|
|
|
// RWX PVCs should not be considered
|
|
Consistently(func() (*corev1.Affinity, error) {
|
|
deployment := &appsv1.Deployment{}
|
|
err := testHelper.GetRelatedResource(pvcViewer, deployment)
|
|
return deployment.Spec.Template.Spec.Affinity, err
|
|
}, duration, interval).Should(BeNil())
|
|
})
|
|
|
|
It("Should generate Affinity on existing RWO", func() {
|
|
By("Creating a RWO PVC and mounting Pod")
|
|
pvcName := "rwo-pvc"
|
|
nodeName := "some-node-mounting-the-rwo-pvc"
|
|
testHelper.CreatePVC(pvcName, corev1.ReadWriteOnce)
|
|
testHelper.CreatePod("rwo-pod", nodeName, pvcName)
|
|
|
|
By("By creating a viewer and referencing a RWO PVC")
|
|
pvcViewer := testHelper.CreateViewer(&kubefloworgv1alpha1.PVCViewerSpec{
|
|
PVC: pvcName,
|
|
PodSpec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "main",
|
|
Image: "busybox",
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{
|
|
Name: "pvc",
|
|
MountPath: "/pvc",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
Volumes: []corev1.Volume{
|
|
{
|
|
Name: "pvc",
|
|
VolumeSource: corev1.VolumeSource{
|
|
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
|
|
ClaimName: pvcName,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
|
|
Eventually(func() (*corev1.Affinity, error) {
|
|
deployment := &appsv1.Deployment{}
|
|
err := testHelper.GetRelatedResource(pvcViewer, deployment)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return deployment.Spec.Template.Spec.Affinity, err
|
|
}, timeout, interval).ShouldNot(And(BeNil(), WithTransform(testHelper.ExtractNodeName, Equal(nodeName))))
|
|
})
|
|
})
|
|
})
|