Merge pull request #126 from fluxcd/capi-kubeconfig

Implement reconciliation for remote clusters provisioned with CAPI
2020-09-30 12:10:09 +03:00 · 2020-09-30 12:10:09 +03:00 · de960c741a
parent ed929826dd 4a283d74b4
commit de960c741a
6 changed files with 197 additions and 16 deletions
--- a/api/v1alpha1/kustomization_types.go
+++ b/api/v1alpha1/kustomization_types.go
@ -44,16 +44,20 @@ type KustomizationSpec struct {
 	// +optional
 	Decryption *Decryption `json:"decryption,omitempty"`

-	// The interval at which to apply the kustomization.
+	// The interval at which to reconcile the kustomization.
 	// +required
 	Interval metav1.Duration `json:"interval"`

+	// The KubeConfig for reconciling the Kustomization on a remote cluster.
+	// +optional
+	KubeConfig *KubeConfig `json:"kubeConfig,omitempty"`
+
 	// Path to the directory containing the kustomization file.
 	// +kubebuilder:validation:Pattern="^\\./"
 	// +required
 	Path string `json:"path"`

-	// Enables garbage collection.
+	// Prune enables garbage collection.
 	// +required
 	Prune bool `json:"prune"`

@ -117,6 +121,16 @@ type Decryption struct {
 	SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"`
 }

+// KubeConfig references a Kubernetes secret generated by CAPI.
+// that contains a kubeconfig file.
+type KubeConfig struct {
+	// The secret name containing a 'value' key
+	// with the kubeconfig file as the value.
+	// Ref: https://github.com/kubernetes-sigs/cluster-api/blob/release-0.3/util/secret/consts.go#L24
+	// +required
+	SecretRef corev1.LocalObjectReference `json:"secretRef,omitempty"`
+}
+
 // KustomizationStatus defines the observed state of a kustomization.
 type KustomizationStatus struct {
 	// ObservedGeneration is the last reconciled generation.
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@ -93,6 +93,22 @@ func (in *Decryption) DeepCopy() *Decryption {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeConfig) DeepCopyInto(out *KubeConfig) {
+	*out = *in
+	out.SecretRef = in.SecretRef
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeConfig.
+func (in *KubeConfig) DeepCopy() *KubeConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Kustomization) DeepCopyInto(out *Kustomization) {
 	*out = *in
@ -166,6 +182,11 @@ func (in *KustomizationSpec) DeepCopyInto(out *KustomizationSpec) {
 		(*in).DeepCopyInto(*out)
 	}
 	out.Interval = in.Interval
+	if in.KubeConfig != nil {
+		in, out := &in.KubeConfig, &out.KubeConfig
+		*out = new(KubeConfig)
+		**out = **in
+	}
 	if in.HealthChecks != nil {
 		in, out := &in.HealthChecks, &out.HealthChecks
 		*out = make([]CrossNamespaceObjectReference, len(*in))
--- a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml
+++ b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml
@ -111,14 +111,28 @@ spec:
                  type: object
                type: array
              interval:
-                description: The interval at which to apply the kustomization.
+                description: The interval at which to reconcile the kustomization.
                type: string
+              kubeConfig:
+                description: The KubeConfig for reconciling the Kustomization on a
+                  remote cluster.
+                properties:
+                  secretRef:
+                    description: 'The secret name containing a ''value'' key with
+                      the kubeconfig file as the value. Ref: https://github.com/kubernetes-sigs/cluster-api/blob/release-0.3/util/secret/consts.go#L24'
+                    properties:
+                      name:
+                        description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                          TODO: Add other useful fields. apiVersion, kind, uid?'
+                        type: string
+                    type: object
+                type: object
              path:
                description: Path to the directory containing the kustomization file.
                pattern: ^\./
                type: string
              prune:
-                description: Enables garbage collection.
+                description: Prune enables garbage collection.
                type: boolean
              serviceAccount:
                description: The Kubernetes service account used for applying the
--- a/controllers/kustomization_controller.go
+++ b/controllers/kustomization_controller.go
@ -465,6 +465,15 @@ func (r *KustomizationReconciler) validate(kustomization kustomizev1.Kustomizati

 	cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp",
 		dirPath, kustomization.GetUID(), kustomization.GetTimeout().String(), kustomization.Spec.Validation)
+
+	if kustomization.Spec.KubeConfig != nil {
+		kubeConfig, err := r.getKubeConfig(kustomization, dirPath)
+		if err != nil {
+			return err
+		}
+		cmd = fmt.Sprintf("%s --kubeconfig=%s", cmd, kubeConfig)
+	}
+
 	command := exec.CommandContext(ctx, "/bin/sh", "-c", cmd)
 	output, err := command.CombinedOutput()
 	if err != nil {
@ -476,6 +485,33 @@ func (r *KustomizationReconciler) validate(kustomization kustomizev1.Kustomizati
 	return nil
 }

+func (r *KustomizationReconciler) getKubeConfig(kustomization kustomizev1.Kustomization, dirPath string) (string, error) {
+	timeout := kustomization.GetTimeout()
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	secretName := types.NamespacedName{
+		Namespace: kustomization.GetNamespace(),
+		Name:      kustomization.Spec.KubeConfig.SecretRef.Name,
+	}
+
+	var secret corev1.Secret
+	if err := r.Get(ctx, secretName, &secret); err != nil {
+		return "", fmt.Errorf("unable to read KubeConfig secret '%s' error: %w", secretName.String(), err)
+	}
+
+	if kubeConfig, ok := secret.Data["value"]; ok {
+		kubeConfigPath := path.Join(dirPath, secretName.Name)
+		if err := ioutil.WriteFile(kubeConfigPath, kubeConfig, os.ModePerm); err != nil {
+			return "", fmt.Errorf("unable to write KubeConfig secret '%s' to storage: %w", secretName.String(), err)
+		}
+	} else {
+		return "", fmt.Errorf("KubeConfig secret '%s' doesn't contain a 'value' key ", secretName.String())
+	}
+
+	return secretName.Name, nil
+}
+
 func (r *KustomizationReconciler) getServiceAccountToken(kustomization kustomizev1.Kustomization) (string, error) {
 	namespacedName := types.NamespacedName{
 		Namespace: kustomization.Spec.ServiceAccount.Namespace,
@ -516,7 +552,7 @@ func (r *KustomizationReconciler) getServiceAccountToken(kustomization kustomize
 	return token, nil
 }

-func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization, revision, dirPath string) (string, error) {
+func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization, dirPath string) (string, error) {
 	start := time.Now()
 	timeout := kustomization.GetTimeout() + (time.Second * 1)
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
@ -525,14 +561,22 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
 	cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --cache-dir=/tmp",
 		dirPath, kustomization.GetUID(), kustomization.Spec.Interval.Duration.String())

-	// impersonate SA
-	if kustomization.Spec.ServiceAccount != nil {
-		saToken, err := r.getServiceAccountToken(kustomization)
+	if kustomization.Spec.KubeConfig != nil {
+		kubeConfig, err := r.getKubeConfig(kustomization, dirPath)
 		if err != nil {
-			return "", fmt.Errorf("service account impersonation failed: %w", err)
+			return "", err
 		}
+		cmd = fmt.Sprintf("%s --kubeconfig=%s", cmd, kubeConfig)
+	} else {
+		// impersonate SA
+		if kustomization.Spec.ServiceAccount != nil {
+			saToken, err := r.getServiceAccountToken(kustomization)
+			if err != nil {
+				return "", fmt.Errorf("service account impersonation failed: %w", err)
+			}

-		cmd = fmt.Sprintf("%s --token %s", cmd, saToken)
+			cmd = fmt.Sprintf("%s --token %s", cmd, saToken)
+		}
 	}

 	command := exec.CommandContext(ctx, "/bin/sh", "-c", cmd)
@ -564,7 +608,7 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
 }

 func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kustomization, revision, dirPath string, delay time.Duration) error {
-	changeSet, err := r.apply(kustomization, revision, dirPath)
+	changeSet, err := r.apply(kustomization, dirPath)
 	if err != nil {
 		// retry apply due to CRD/CR race
 		if strings.Contains(err.Error(), "could not find the requested resource") ||
@ -573,7 +617,7 @@ func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kusto
 				"error", err.Error(),
 				"kustomization", fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()))
 			time.Sleep(delay)
-			if changeSet, err := r.apply(kustomization, revision, dirPath); err != nil {
+			if changeSet, err := r.apply(kustomization, dirPath); err != nil {
 				return err
 			} else {
 				if changeSet != "" {
@ -592,6 +636,14 @@ func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kusto
 }

 func (r *KustomizationReconciler) prune(kustomization kustomizev1.Kustomization, snapshot *kustomizev1.Snapshot, force bool) error {
+	if kustomization.Spec.KubeConfig != nil {
+		// TODO: implement pruning for remote clusters
+		r.Log.WithValues(
+			strings.ToLower(kustomization.Kind),
+			fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
+		).V(2).Info("skipping pruning, garbage collection is not implemented for remote clusters")
+		return nil
+	}
 	if kustomization.Status.Snapshot == nil || snapshot == nil {
 		return nil
 	}
--- a/docs/api/kustomize.md
+++ b/docs/api/kustomize.md
@ -108,7 +108,21 @@ Kubernetes meta/v1.Duration
 </em>
 </td>
 <td>
-<p>The interval at which to apply the kustomization.</p>
+<p>The interval at which to reconcile the kustomization.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>kubeConfig</code><br>
+<em>
+<a href="#kustomize.toolkit.fluxcd.io/v1alpha1.KubeConfig">
+KubeConfig
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The KubeConfig for reconciling the Kustomization on a remote cluster.</p>
 </td>
 </tr>
 <tr>
@ -130,7 +144,7 @@ bool
 </em>
 </td>
 <td>
-<p>Enables garbage collection.</p>
+<p>Prune enables garbage collection.</p>
 </td>
 </tr>
 <tr>
@ -513,6 +527,43 @@ Kubernetes core/v1.LocalObjectReference
 </table>
 </div>
 </div>
+<h3 id="kustomize.toolkit.fluxcd.io/v1alpha1.KubeConfig">KubeConfig
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#kustomize.toolkit.fluxcd.io/v1alpha1.KustomizationSpec">KustomizationSpec</a>)
+</p>
+<p>KubeConfig references a Kubernetes secret generated by CAPI.
+that contains a kubeconfig file.</p>
+<div class="md-typeset__scrollwrap">
+<div class="md-typeset__table">
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>secretRef</code><br>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#localobjectreference-v1-core">
+Kubernetes core/v1.LocalObjectReference
+</a>
+</em>
+</td>
+<td>
+<p>The secret name containing a &lsquo;value&rsquo; key
+with the kubeconfig file as the value.
+Ref: <a href="https://github.com/kubernetes-sigs/cluster-api/blob/release-0.3/util/secret/consts.go#L24">https://github.com/kubernetes-sigs/cluster-api/blob/release-0.3/util/secret/consts.go#L24</a></p>
+</td>
+</tr>
+</tbody>
+</table>
+</div>
+</div>
 <h3 id="kustomize.toolkit.fluxcd.io/v1alpha1.KustomizationSpec">KustomizationSpec
 </h3>
 <p>
@ -570,7 +621,21 @@ Kubernetes meta/v1.Duration
 </em>
 </td>
 <td>
-<p>The interval at which to apply the kustomization.</p>
+<p>The interval at which to reconcile the kustomization.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>kubeConfig</code><br>
+<em>
+<a href="#kustomize.toolkit.fluxcd.io/v1alpha1.KubeConfig">
+KubeConfig
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>The KubeConfig for reconciling the Kustomization on a remote cluster.</p>
 </td>
 </tr>
 <tr>
@ -592,7 +657,7 @@ bool
 </em>
 </td>
 <td>
-<p>Enables garbage collection.</p>
+<p>Prune enables garbage collection.</p>
 </td>
 </tr>
 <tr>
--- a/docs/spec/v1alpha1/kustomization.md
+++ b/docs/spec/v1alpha1/kustomization.md
@ -25,6 +25,10 @@ type KustomizationSpec struct {
 	// +required
 	Interval metav1.Duration `json:"interval"`

+	// The KubeConfig for reconciling the Kustomization on a remote cluster.
+	// +optional
+	KubeConfig *KubeConfig `json:"kubeConfig,omitempty"`
+
 	// Path to the directory containing the kustomization file.
 	// +kubebuilder:validation:Pattern="^\\./"
 	// +required
@ -84,6 +88,17 @@ type Decryption struct {
 }
 ```

+KubeConfig references a Kubernetes secret generated by CAPI:
+
+```go
+type KubeConfig struct {
+	// The secret name containing a 'value' key with the kubeconfig file as the value.
+	// Ref: https://github.com/kubernetes-sigs/cluster-api/blob/release-0.3/util/secret/consts.go#L24
+	// +required
+	SecretRef corev1.LocalObjectReference `json:"secretRef,omitempty"`
+}
+```
+
 The status sub-resource records the result of the last reconciliation:

 ```go