Introduce health assessment

- add health checks to API
- implement rollout status check

api/v1alpha1/kustomization_types.go

@@ -37,6 +37,11 @@ type KustomizationSpec struct {
 	// +optional
 	Prune string `json:"prune,omitempty"`
 
+	// A list of workloads for health assessment.
+	// Checks the rollout status of each supplied Deployment, DaemonSet or StatefulSet.
+	// +optional
+	HealthChecks []WorkloadReference `json:"healthChecks,omitempty"`
+
 	// Reference of the source where the kustomization file is.
 	// +required
 	SourceRef corev1.TypedLocalObjectReference `json:"sourceRef"`
@@ -53,6 +58,22 @@ type KustomizationSpec struct {
 	Validation string `json:"validation,omitempty"`
 }
 
+// WorkloadReference defines a reference to Deployment, DaemonSet or StatefulSet
+type WorkloadReference struct {
+	// Kind is the type of resource being referenced
+	// +kubebuilder:validation:Enum=Deployment;DaemonSet;StatefulSet
+	// +required
+	Kind string `json:"kind"`
+
+	// Name is the name of resource being referenced
+	// +required
+	Name string `json:"name"`
+
+	// Namespace is the namespace of resource being referenced
+	// +optional
+	Namespace string `json:"namespace,omitempty"`
+}
+
 // KustomizationStatus defines the observed state of a kustomization.
 type KustomizationStatus struct {
 	// +optional

api/v1alpha1/zz_generated.deepcopy.go

@@ -103,6 +103,11 @@ func (in *KustomizationList) DeepCopyObject() runtime.Object {
 func (in *KustomizationSpec) DeepCopyInto(out *KustomizationSpec) {
 	*out = *in
 	out.Interval = in.Interval
+	if in.HealthChecks != nil {
+		in, out := &in.HealthChecks, &out.HealthChecks
+		*out = make([]WorkloadReference, len(*in))
+		copy(*out, *in)
+	}
 	in.SourceRef.DeepCopyInto(&out.SourceRef)
 }
 
@@ -137,3 +142,18 @@ func (in *KustomizationStatus) DeepCopy() *KustomizationStatus {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WorkloadReference) DeepCopyInto(out *WorkloadReference) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadReference.
+func (in *WorkloadReference) DeepCopy() *WorkloadReference {
+	if in == nil {
+		return nil
+	}
+	out := new(WorkloadReference)
+	in.DeepCopyInto(out)
+	return out
+}

config/crd/bases/kustomize.fluxcd.io_kustomizations.yaml

@@ -46,6 +46,31 @@ spec:
         spec:
           description: KustomizationSpec defines the desired state of a kustomization.
           properties:
+            healthChecks:
+              description: A list of workloads for health assessment. Checks the rollout
+                status of each supplied Deployment, DaemonSet or StatefulSet.
+              items:
+                description: WorkloadReference defines a reference to Deployment,
+                  DaemonSet or StatefulSet
+                properties:
+                  kind:
+                    description: Kind is the type of resource being referenced
+                    enum:
+                    - Deployment
+                    - DaemonSet
+                    - StatefulSet
+                    type: string
+                  name:
+                    description: Name is the name of resource being referenced
+                    type: string
+                  namespace:
+                    description: Namespace is the namespace of resource being referenced
+                    type: string
+                required:
+                - kind
+                - name
+                type: object
+              type: array
             interval:
               description: The interval at which to apply the kustomization.
               type: string

config/samples/kustomize_v1alpha1_kustomization.yaml

@@ -10,6 +10,10 @@ spec:
   sourceRef:
     kind: GitRepository
     name: podinfo
+  healthChecks:
+    - kind: Deployment
+      name: podinfo
+      namespace: dev
 ---
 apiVersion: kustomize.fluxcd.io/v1alpha1
 kind: Kustomization

controllers/kustomization_controller.go

@@ -234,6 +234,15 @@ func (r *KustomizationReconciler) sync(
 		fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
 	).Info(applyDuration, "output", r.parseApplyOutput(output))
 
+	err = r.isHealthy(kustomization)
+	if err != nil {
+		return kustomizev1.KustomizationNotReady(
+			kustomization,
+			kustomizev1.ApplyFailedReason,
+			err.Error(),
+		), err
+	}
+
 	return kustomizev1.KustomizationReady(
 		kustomization,
 		kustomizev1.ApplySucceedReason,
@@ -267,3 +276,26 @@ func (r *KustomizationReconciler) parseApplyOutput(in []byte) map[string]string
 	}
 	return result
 }
+
+func (r *KustomizationReconciler) isHealthy(kustomization kustomizev1.Kustomization) error {
+	timeout := kustomization.Spec.Interval.Duration + (time.Second * 1)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	for _, check := range kustomization.Spec.HealthChecks {
+		cmd := fmt.Sprintf("kubectl -n %s rollout status %s %s --timeout=%s",
+			check.Namespace, check.Kind, check.Name, kustomization.Spec.Interval.Duration.String())
+		command := exec.CommandContext(ctx, "/bin/sh", "-c", cmd)
+		output, err := command.CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("health check failed for %s %s/%s: %s",
+				check.Kind, check.Namespace, check.Name, string(output))
+		} else {
+			r.Log.WithValues(
+				strings.ToLower(kustomization.Kind),
+				fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
+			).Info(fmt.Sprintf("health check passed for %s %s/%s",
+				check.Kind, check.Namespace, check.Name))
+		}
+	}
+	return nil
+}