From 0dae17ffd6df5bf0549f3081b88e9b39fc181b0d Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Sun, 19 Apr 2020 01:28:58 +0300 Subject: [PATCH] Introduce health assessment - add health checks to API - implement rollout status check --- api/v1alpha1/kustomization_types.go | 21 ++++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 20 ++++++++++++ .../kustomize.fluxcd.io_kustomizations.yaml | 25 +++++++++++++++ .../kustomize_v1alpha1_kustomization.yaml | 4 +++ controllers/kustomization_controller.go | 32 +++++++++++++++++++ 5 files changed, 102 insertions(+) diff --git a/api/v1alpha1/kustomization_types.go b/api/v1alpha1/kustomization_types.go index 9def1ba..5a560f2 100644 --- a/api/v1alpha1/kustomization_types.go +++ b/api/v1alpha1/kustomization_types.go @@ -37,6 +37,11 @@ type KustomizationSpec struct { // +optional Prune string `json:"prune,omitempty"` + // A list of workloads for health assessment. + // Checks the rollout status of each supplied Deployment, DaemonSet or StatefulSet. + // +optional + HealthChecks []WorkloadReference `json:"healthChecks,omitempty"` + // Reference of the source where the kustomization file is. // +required SourceRef corev1.TypedLocalObjectReference `json:"sourceRef"` @@ -53,6 +58,22 @@ type KustomizationSpec struct { Validation string `json:"validation,omitempty"` } +// WorkloadReference defines a reference to Deployment, DaemonSet or StatefulSet +type WorkloadReference struct { + // Kind is the type of resource being referenced + // +kubebuilder:validation:Enum=Deployment;DaemonSet;StatefulSet + // +required + Kind string `json:"kind"` + + // Name is the name of resource being referenced + // +required + Name string `json:"name"` + + // Namespace is the namespace of resource being referenced + // +optional + Namespace string `json:"namespace,omitempty"` +} + // KustomizationStatus defines the observed state of a kustomization. type KustomizationStatus struct { // +optional diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2fe36c1..6c6a2c4 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -103,6 +103,11 @@ func (in *KustomizationList) DeepCopyObject() runtime.Object { func (in *KustomizationSpec) DeepCopyInto(out *KustomizationSpec) { *out = *in out.Interval = in.Interval + if in.HealthChecks != nil { + in, out := &in.HealthChecks, &out.HealthChecks + *out = make([]WorkloadReference, len(*in)) + copy(*out, *in) + } in.SourceRef.DeepCopyInto(&out.SourceRef) } @@ -137,3 +142,18 @@ func (in *KustomizationStatus) DeepCopy() *KustomizationStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadReference) DeepCopyInto(out *WorkloadReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadReference. +func (in *WorkloadReference) DeepCopy() *WorkloadReference { + if in == nil { + return nil + } + out := new(WorkloadReference) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/kustomize.fluxcd.io_kustomizations.yaml b/config/crd/bases/kustomize.fluxcd.io_kustomizations.yaml index 3b273e3..aef3021 100644 --- a/config/crd/bases/kustomize.fluxcd.io_kustomizations.yaml +++ b/config/crd/bases/kustomize.fluxcd.io_kustomizations.yaml @@ -46,6 +46,31 @@ spec: spec: description: KustomizationSpec defines the desired state of a kustomization. properties: + healthChecks: + description: A list of workloads for health assessment. Checks the rollout + status of each supplied Deployment, DaemonSet or StatefulSet. + items: + description: WorkloadReference defines a reference to Deployment, + DaemonSet or StatefulSet + properties: + kind: + description: Kind is the type of resource being referenced + enum: + - Deployment + - DaemonSet + - StatefulSet + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource being referenced + type: string + required: + - kind + - name + type: object + type: array interval: description: The interval at which to apply the kustomization. type: string diff --git a/config/samples/kustomize_v1alpha1_kustomization.yaml b/config/samples/kustomize_v1alpha1_kustomization.yaml index b32fb56..1f8bb95 100644 --- a/config/samples/kustomize_v1alpha1_kustomization.yaml +++ b/config/samples/kustomize_v1alpha1_kustomization.yaml @@ -10,6 +10,10 @@ spec: sourceRef: kind: GitRepository name: podinfo + healthChecks: + - kind: Deployment + name: podinfo + namespace: dev --- apiVersion: kustomize.fluxcd.io/v1alpha1 kind: Kustomization diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go index 9efe5f1..ec69e0e 100644 --- a/controllers/kustomization_controller.go +++ b/controllers/kustomization_controller.go @@ -234,6 +234,15 @@ func (r *KustomizationReconciler) sync( fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()), ).Info(applyDuration, "output", r.parseApplyOutput(output)) + err = r.isHealthy(kustomization) + if err != nil { + return kustomizev1.KustomizationNotReady( + kustomization, + kustomizev1.ApplyFailedReason, + err.Error(), + ), err + } + return kustomizev1.KustomizationReady( kustomization, kustomizev1.ApplySucceedReason, @@ -267,3 +276,26 @@ func (r *KustomizationReconciler) parseApplyOutput(in []byte) map[string]string } return result } + +func (r *KustomizationReconciler) isHealthy(kustomization kustomizev1.Kustomization) error { + timeout := kustomization.Spec.Interval.Duration + (time.Second * 1) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + for _, check := range kustomization.Spec.HealthChecks { + cmd := fmt.Sprintf("kubectl -n %s rollout status %s %s --timeout=%s", + check.Namespace, check.Kind, check.Name, kustomization.Spec.Interval.Duration.String()) + command := exec.CommandContext(ctx, "/bin/sh", "-c", cmd) + output, err := command.CombinedOutput() + if err != nil { + return fmt.Errorf("health check failed for %s %s/%s: %s", + check.Kind, check.Namespace, check.Name, string(output)) + } else { + r.Log.WithValues( + strings.ToLower(kustomization.Kind), + fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()), + ).Info(fmt.Sprintf("health check passed for %s %s/%s", + check.Kind, check.Namespace, check.Name)) + } + } + return nil +}