Merge pull request #4867 from veophi/bugfix/deployment-generation

Align federated Deployment's observedGeneration semantics with its native

pkg/apis/work/v1alpha2/well_known_constants.go

@@ -60,6 +60,17 @@ const (
 
 	// BindingManagedByLabel is added to ResourceBinding to represent what kind of resource manages this Binding.
 	BindingManagedByLabel = "binding.karmada.io/managed-by"
+
+	// ResourceTemplateGenerationAnnotationKey records the generation of resource template in Karmada APIServer,
+	// It will be injected into the resource when propagating to member clusters, to denote the specific version of
+	// the resource template from which the resource is derived. It might be helpful in the following cases:
+	// 1. Facilitating observation from member clusters to ascertain if the most recent resource template has been
+	//    completely synced.
+	// 2. The annotation will be synced back to Karmada during the process of syncing resource status,
+	//    by leveraging this annotation, Karmada can infer if the most recent resource template has been completely
+	//    synced on member clusters, then generates accurate observed generation(like Deployment's .status.observedGeneration)
+	//    which might be required by the release system.
+	ResourceTemplateGenerationAnnotationKey = "resourcetemplate.karmada.io/generation"
 )
 
 // Define resource conflict resolution

pkg/controllers/binding/common.go

@@ -17,6 +17,8 @@ limitations under the License.
 package binding
 
 import (
+	"strconv"
+
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -168,6 +170,9 @@ func mergeLabel(workload *unstructured.Unstructured, binding metav1.Object, scop
 
 func mergeAnnotations(workload *unstructured.Unstructured, binding metav1.Object, scope apiextensionsv1.ResourceScope) map[string]string {
 	annotations := make(map[string]string)
+	if workload.GetGeneration() > 0 {
+		util.MergeAnnotation(workload, workv1alpha2.ResourceTemplateGenerationAnnotationKey, strconv.FormatInt(workload.GetGeneration(), 10))
+	}
 
 	if scope == apiextensionsv1.NamespaceScoped {
 		util.MergeAnnotation(workload, workv1alpha2.ResourceBindingNamespaceAnnotationKey, binding.GetNamespace())

pkg/resourceinterpreter/default/native/aggregatestatus.go

@@ -65,27 +65,39 @@ func aggregateDeploymentStatus(object *unstructured.Unstructured, aggregatedStat
 
 	oldStatus := &deploy.Status
 	newStatus := &appsv1.DeploymentStatus{}
+	observedLatestResourceTemplateGenerationCount := 0
 	for _, item := range aggregatedStatusItems {
 		if item.Status == nil {
 			continue
 		}
-		temp := &appsv1.DeploymentStatus{}
+		member := &WrappedDeploymentStatus{}
-		if err = json.Unmarshal(item.Status.Raw, temp); err != nil {
+		if err = json.Unmarshal(item.Status.Raw, member); err != nil {
 			return nil, err
 		}
 		klog.V(3).Infof("Grab deployment(%s/%s) status from cluster(%s), replicas: %d, ready: %d, updated: %d, available: %d, unavailable: %d",
-			deploy.Namespace, deploy.Name, item.ClusterName, temp.Replicas, temp.ReadyReplicas, temp.UpdatedReplicas, temp.AvailableReplicas, temp.UnavailableReplicas)
+			deploy.Namespace, deploy.Name, item.ClusterName, member.Replicas, member.ReadyReplicas, member.UpdatedReplicas, member.AvailableReplicas, member.UnavailableReplicas)
 
-		// always set 'observedGeneration' with current generation(.metadata.generation)
+		// `memberStatus.ObservedGeneration >= memberStatus.Generation` means the member's status corresponds the latest spec revision of the member deployment.
-		// which is the generation Karmada 'observed'.
+		// `memberStatus.ResourceTemplateGeneration >= deploy.Generation` means the member deployment has been aligned with the latest spec revision of federated deployment.
-		// The 'observedGeneration' is mainly used by GitOps tools(like 'Argo CD') to assess the health status.
+		// If both conditions are met, we consider the member's status corresponds the latest spec revision of federated deployment.
-		// For more details, please refer to https://argo-cd.readthedocs.io/en/stable/operator-manual/health/.
+		if member.ObservedGeneration >= member.Generation &&
+			member.ResourceTemplateGeneration >= deploy.Generation {
+			observedLatestResourceTemplateGenerationCount++
+		}
+
+		newStatus.Replicas += member.Replicas
+		newStatus.ReadyReplicas += member.ReadyReplicas
+		newStatus.UpdatedReplicas += member.UpdatedReplicas
+		newStatus.AvailableReplicas += member.AvailableReplicas
+		newStatus.UnavailableReplicas += member.UnavailableReplicas
+	}
+
+	// The 'observedGeneration' is mainly used by GitOps tools(like 'Argo CD') to assess the health status.
+	// For more details, please refer to https://argo-cd.readthedocs.io/en/stable/operator-manual/health/.
+	if observedLatestResourceTemplateGenerationCount == len(aggregatedStatusItems) {
 		newStatus.ObservedGeneration = deploy.Generation
-		newStatus.Replicas += temp.Replicas
+	} else {
-		newStatus.ReadyReplicas += temp.ReadyReplicas
+		newStatus.ObservedGeneration = oldStatus.ObservedGeneration
-		newStatus.UpdatedReplicas += temp.UpdatedReplicas
-		newStatus.AvailableReplicas += temp.AvailableReplicas
-		newStatus.UnavailableReplicas += temp.UnavailableReplicas
 	}
 
 	if oldStatus.ObservedGeneration == newStatus.ObservedGeneration &&

pkg/resourceinterpreter/default/native/reflectstatus.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/klog/v2"
 
+	"github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
 	"github.com/karmada-io/karmada/pkg/util"
 	"github.com/karmada-io/karmada/pkg/util/helper"
 )
@@ -68,12 +69,25 @@ func reflectDeploymentStatus(object *unstructured.Unstructured) (*runtime.RawExt
 		return nil, fmt.Errorf("failed to convert DeploymentStatus from map[string]interface{}: %v", err)
 	}
 
-	grabStatus := appsv1.DeploymentStatus{
+	resourceTemplateGenerationInt := int64(0)
-		Replicas:            deploymentStatus.Replicas,
+	resourceTemplateGenerationStr := util.GetAnnotationValue(object.GetAnnotations(), v1alpha2.ResourceTemplateGenerationAnnotationKey)
-		UpdatedReplicas:     deploymentStatus.UpdatedReplicas,
+	err = runtime.Convert_string_To_int64(&resourceTemplateGenerationStr, &resourceTemplateGenerationInt, nil)
-		ReadyReplicas:       deploymentStatus.ReadyReplicas,
+	if err != nil {
-		AvailableReplicas:   deploymentStatus.AvailableReplicas,
+		klog.Errorf("Failed to parse Deployment(%s/%s) generation from annotation(%s:%s): %v", object.GetNamespace(), object.GetName(), v1alpha2.ResourceTemplateGenerationAnnotationKey, resourceTemplateGenerationStr, err)
-		UnavailableReplicas: deploymentStatus.UnavailableReplicas,
+		return nil, err
+	}
+
+	grabStatus := &WrappedDeploymentStatus{
+		Generation:                 object.GetGeneration(),
+		ResourceTemplateGeneration: resourceTemplateGenerationInt,
+		DeploymentStatus: appsv1.DeploymentStatus{
+			Replicas:            deploymentStatus.Replicas,
+			UpdatedReplicas:     deploymentStatus.UpdatedReplicas,
+			ReadyReplicas:       deploymentStatus.ReadyReplicas,
+			AvailableReplicas:   deploymentStatus.AvailableReplicas,
+			UnavailableReplicas: deploymentStatus.UnavailableReplicas,
+			ObservedGeneration:  deploymentStatus.ObservedGeneration,
+		},
 	}
 
 	grabStatusRaw, err := helper.BuildStatusRawExtension(grabStatus)

pkg/resourceinterpreter/default/native/status_type.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2024 The Karmada Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package native
+
+import appsv1 "k8s.io/api/apps/v1"
+
+// WrappedDeploymentStatus is a wrapper for appsv1.DeploymentStatus.
+type WrappedDeploymentStatus struct {
+	appsv1.DeploymentStatus `json:",inline"`
+
+	// Generation holds the generation(.metadata.generation) of resource running on member cluster.
+	Generation int64 `json:"generation,omitempty"`
+	// ResourceTemplateGeneration holds the value of annotation resourcetemplate.karmada.io/generation grabbed
+	// from resource running on member cluster.
+	ResourceTemplateGeneration int64 `json:"resourceTemplateGeneration,omitempty"`
+}