propagate dependencies support propagate pvc

Signed-off-by: hanweisen <hanweisen_yewu@cmss.chinamobile.com>

pkg/dependenciesdistributor/dependencies_distributor.go

@@ -49,6 +49,7 @@ const (
 var supportedTypes = []schema.GroupVersionResource{
 	corev1.SchemeGroupVersion.WithResource("configmaps"),
 	corev1.SchemeGroupVersion.WithResource("secrets"),
+	corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims"),
 }
 
 // DependenciesDistributor is to automatically propagate relevant resources.

pkg/resourceinterpreter/defaultinterpreter/dependencies.go

@@ -117,6 +117,7 @@ func getDependenciesFromPodTemplate(podObj *corev1.Pod) ([]configv1alpha1.Depend
 	dependentConfigMaps := getConfigMapNames(podObj)
 	dependentSecrets := getSecretNames(podObj)
 	dependentSas := getServiceAccountNames(podObj)
+	dependentPVCs := getPVCNames(podObj)
 	var dependentObjectRefs []configv1alpha1.DependentObjectReference
 	for cm := range dependentConfigMaps {
 		dependentObjectRefs = append(dependentObjectRefs, configv1alpha1.DependentObjectReference{
@@ -143,6 +144,16 @@ func getDependenciesFromPodTemplate(podObj *corev1.Pod) ([]configv1alpha1.Depend
 			Name:       sa,
 		})
 	}
+
+	for pvc := range dependentPVCs {
+		dependentObjectRefs = append(dependentObjectRefs, configv1alpha1.DependentObjectReference{
+			APIVersion: "v1",
+			Kind:       "PersistentVolumeClaim",
+			Namespace:  podObj.Namespace,
+			Name:       pvc,
+		})
+	}
+
 	return dependentObjectRefs, nil
 }
 
@@ -171,3 +182,17 @@ func getConfigMapNames(pod *corev1.Pod) sets.String {
 	})
 	return result
 }
+
+func getPVCNames(pod *corev1.Pod) sets.String {
+	result := sets.NewString()
+	for i := range pod.Spec.Volumes {
+		volume := pod.Spec.Volumes[i]
+		if volume.PersistentVolumeClaim != nil {
+			claimName := volume.PersistentVolumeClaim.ClaimName
+			if len(claimName) != 0 {
+				result.Insert(claimName)
+			}
+		}
+	}
+	return result
+}

pkg/resourceinterpreter/defaultinterpreter/dependencies_test.go

@@ -101,6 +101,51 @@ func TestGetConfigMapNames(t *testing.T) {
 	}
 }
 
+func TestGetPVCNames(t *testing.T) {
+	fakePod := helper.NewPod("foo", "bar")
+	fakePod.Spec.Volumes = []corev1.Volume{
+		{
+			Name: "foo-name",
+			VolumeSource: corev1.VolumeSource{
+				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+					ClaimName: "fake-foo",
+					ReadOnly:  true,
+				},
+			},
+		},
+		{
+			Name: "bar-name",
+			VolumeSource: corev1.VolumeSource{
+				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+					ClaimName: "fake-bar",
+					ReadOnly:  true,
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		name     string
+		pod      *corev1.Pod
+		expected sets.String
+	}{
+		{
+			name:     "get pvc names from pod",
+			pod:      fakePod,
+			expected: sets.NewString("fake-foo", "fake-bar"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			res := getPVCNames(tt.pod)
+			if !reflect.DeepEqual(res, tt.expected) {
+				t.Errorf("getPVCNames() = %v, want %v", res, tt.expected)
+			}
+		})
+	}
+}
+
 func TestGetDependenciesFromPodTemplate(t *testing.T) {
 	fakePod := helper.NewPod("foo", "bar")
 	fakePod.Spec.Volumes = []corev1.Volume{

test/e2e/dependenciesdistributor_test.go

@@ -4,6 +4,7 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/rand"
 
@@ -201,5 +202,80 @@ var _ = ginkgo.Describe("[DependenciesDistributor] automatically propagate relev
 				})
 			})
 		})
+
+		ginkgo.When("persistentVolumeClaim propagate automatically", func() {
+			var pvcName string
+			var pvc *corev1.PersistentVolumeClaim
+
+			ginkgo.BeforeEach(func() {
+				pvcName = pvcNamePrefix + rand.String(RandomStrLength)
+				pvc = testhelper.NewPVC(testNamespace, pvcName, corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceStorage: resource.MustParse("1Gi"),
+					},
+				}, corev1.ReadWriteOnce)
+
+				volumes := []corev1.Volume{{
+					Name: "vol-pvc",
+					VolumeSource: corev1.VolumeSource{
+						PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+							ClaimName: pvc.GetName(),
+							ReadOnly:  false,
+						}}}}
+				deployment = testhelper.NewDeploymentWithVolumes(testNamespace, deploymentName, volumes)
+
+				policy = testhelper.NewPropagationPolicy(testNamespace, policyName, []policyv1alpha1.ResourceSelector{
+					{
+						APIVersion: deployment.APIVersion,
+						Kind:       deployment.Kind,
+						Name:       deployment.Name,
+					},
+				}, policyv1alpha1.Placement{
+					ClusterAffinity: &policyv1alpha1.ClusterAffinity{
+						ClusterNames: initClusterNames,
+					},
+				})
+				policy.Spec.PropagateDeps = true
+			})
+
+			ginkgo.BeforeEach(func() {
+				framework.CreatePVC(kubeClient, pvc)
+				ginkgo.DeferCleanup(func() {
+					framework.RemovePVC(kubeClient, pvc.GetNamespace(), pvc.GetName())
+				})
+			})
+
+			ginkgo.It("persistentVolumeClaim automatically propagation testing", func() {
+				ginkgo.By("check if the persistentVolumeClaim is propagated automatically", func() {
+					framework.WaitDeploymentPresentOnClustersFitWith(initClusterNames, deployment.Namespace, deployment.Name,
+						func(deployment *appsv1.Deployment) bool {
+							return true
+						})
+
+					framework.WaitPVCPresentOnClustersFitWith(initClusterNames, pvc.GetNamespace(), pvc.GetName(),
+						func(pvc *corev1.PersistentVolumeClaim) bool {
+							return true
+						})
+				})
+
+				ginkgo.By("make the persistentVolumeClaim is not referenced by the deployment ", func() {
+					updateVolumes := []corev1.Volume{
+						{
+							Name: "vol-configmap",
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "configMap-test",
+									},
+								},
+							},
+						},
+					}
+
+					framework.UpdateDeploymentVolumes(kubeClient, deployment, updateVolumes)
+					framework.WaitPVCDisappearOnClusters(initClusterNames, pvc.GetNamespace(), pvc.GetName())
+				})
+			})
+		})
 	})
 })

test/e2e/framework/persistentvolumeclaim.go

@@ -0,0 +1,83 @@
+package framework
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+)
+
+// CreatePVC create PersistentVolumeClaim.
+func CreatePVC(client kubernetes.Interface, pvc *corev1.PersistentVolumeClaim) {
+	ginkgo.By(fmt.Sprintf("Creating PersistentVolumeClaim(%s/%s)", pvc.Namespace, pvc.Name), func() {
+		_, err := client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(context.TODO(), pvc, metav1.CreateOptions{})
+		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+	})
+}
+
+// RemovePVC delete PersistentVolumeClaim.
+func RemovePVC(client kubernetes.Interface, namespace, name string) {
+	ginkgo.By(fmt.Sprintf("Removing PersistentVolumeClaim(%s/%s)", namespace, name), func() {
+		err := client.CoreV1().PersistentVolumeClaims(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
+		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+	})
+}
+
+// WaitPVCPresentOnClustersFitWith wait PersistentVolumeClaim present on clusters sync with fit func.
+func WaitPVCPresentOnClustersFitWith(clusters []string, namespace, name string, fit func(pvc *corev1.PersistentVolumeClaim) bool) {
+	ginkgo.By(fmt.Sprintf("Waiting for PersistentVolumeClaim(%s/%s) synced on member clusters", namespace, name), func() {
+		for _, clusterName := range clusters {
+			WaitPVCPresentOnClusterFitWith(clusterName, namespace, name, fit)
+		}
+	})
+}
+
+// WaitPVCPresentOnClusterFitWith wait PersistentVolumeClaim present on member cluster sync with fit func.
+func WaitPVCPresentOnClusterFitWith(cluster, namespace, name string, fit func(pvc *corev1.PersistentVolumeClaim) bool) {
+	clusterClient := GetClusterClient(cluster)
+	gomega.Expect(clusterClient).ShouldNot(gomega.BeNil())
+
+	klog.Infof("Waiting for PersistentVolumeClaim(%s/%s) synced on cluster(%s)", namespace, name, cluster)
+	gomega.Eventually(func() bool {
+		pvc, err := clusterClient.CoreV1().PersistentVolumeClaims(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			return false
+		}
+		return fit(pvc)
+	}, pollTimeout, pollInterval).Should(gomega.Equal(true))
+}
+
+// WaitPVCDisappearOnCluster wait PersistentVolumeClaim disappear on cluster until timeout.
+func WaitPVCDisappearOnCluster(cluster, namespace, name string) {
+	clusterClient := GetClusterClient(cluster)
+	gomega.Expect(clusterClient).ShouldNot(gomega.BeNil())
+
+	klog.Infof("Waiting for PersistentVolumeClaim(%s/%s) disappear on cluster(%s)", namespace, name, cluster)
+	gomega.Eventually(func() bool {
+		_, err := clusterClient.CoreV1().PersistentVolumeClaims(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+		if err == nil {
+			return false
+		}
+		if apierrors.IsNotFound(err) {
+			return true
+		}
+
+		klog.Errorf("Failed to get PersistentVolumeClaim(%s/%s) on cluster(%s), err: %v", namespace, name, cluster, err)
+		return false
+	}, pollTimeout, pollInterval).Should(gomega.Equal(true))
+}
+
+// WaitPVCDisappearOnClusters Wait for the PersistentVolumeClaim to disappear on member clusters until timeout.
+func WaitPVCDisappearOnClusters(clusters []string, namespace, name string) {
+	ginkgo.By(fmt.Sprintf("Check if PersistentVolumeClaim(%s/%s) diappears on member clusters", namespace, name), func() {
+		for _, clusterName := range clusters {
+			WaitPVCDisappearOnCluster(clusterName, namespace, name)
+		}
+	})
+}

test/e2e/suite_test.go

@@ -46,6 +46,7 @@ const (
 	federatedResourceQuotaPrefix = "frq-"
 	configMapNamePrefix          = "configmap-"
 	secretNamePrefix             = "secret-"
+	pvcNamePrefix                = "pvc-"
 	ingressNamePrefix            = "ingress-"
 	daemonSetNamePrefix          = "daemonset-"
 	statefulSetNamePrefix        = "statefulset-"

test/helper/resource.go

@@ -589,6 +589,17 @@ func NewConfigMap(namespace string, name string, data map[string]string) *corev1
 	}
 }
 
+// NewPVC will build a new PersistentVolumeClaim.
+func NewPVC(namespace, name string, resources corev1.ResourceRequirements, accessModes ...corev1.PersistentVolumeAccessMode) *corev1.PersistentVolumeClaim {
+	return &corev1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
+		Spec: corev1.PersistentVolumeClaimSpec{
+			AccessModes: accessModes,
+			Resources:   resources,
+		},
+	}
+}
+
 // NewServiceaccount will build a new serviceaccount.
 func NewServiceaccount(namespace, name string) *corev1.ServiceAccount {
 	return &corev1.ServiceAccount{