diff --git a/api/v1alpha1/helmchart_types.go b/api/v1alpha1/helmchart_types.go index 03abf07e..f58f9be3 100644 --- a/api/v1alpha1/helmchart_types.go +++ b/api/v1alpha1/helmchart_types.go @@ -25,25 +25,41 @@ const HelmChartKind = "HelmChart" // HelmChartSpec defines the desired state of a Helm chart. type HelmChartSpec struct { - // The name of the Helm chart, as made available by the referenced - // Helm repository. + // The name or path the Helm chart is available at in the SourceRef. // +required - Name string `json:"name"` + Chart string `json:"chart"` - // The chart version semver expression, defaults to latest when - // omitted. + // The chart version semver expression, ignored for charts from GitRepository + // sources. Defaults to latest when omitted. // +optional Version string `json:"version,omitempty"` - // The name of the HelmRepository the chart is available at. + // The reference to the Source the chart is available at. // +required - HelmRepositoryRef corev1.LocalObjectReference `json:"helmRepositoryRef"` + SourceRef LocalHelmChartSourceReference `json:"sourceRef"` - // The interval at which to check the Helm repository for updates. + // The interval at which to check the Source for updates. // +required Interval metav1.Duration `json:"interval"` } +// LocalHelmChartSourceReference contains enough information to let you locate the +// typed referenced object at namespace level. +type LocalHelmChartSourceReference struct { + // APIVersion of the referent. + // +optional + APIVersion string `json:"apiVersion,omitempty"` + + // Kind of the referent, valid values are ('HelmRepository', 'GitRepository'). + // +kubebuilder:validation:Enum=HelmRepository;GitRepository + // +required + Kind string `json:"kind"` + + // Name of the referent. + // +required + Name string `json:"name"` +} + // HelmChartStatus defines the observed state of the HelmChart. type HelmChartStatus struct { // +optional @@ -63,72 +79,62 @@ const ( // Helm chart failed. ChartPullFailedReason string = "ChartPullFailed" - // ChartPulLSucceededReason represents the fact that the pull of + // ChartPullSucceededReason represents the fact that the pull of // the Helm chart succeeded. ChartPullSucceededReason string = "ChartPullSucceeded" + + // ChartPackageFailedReason represent the fact that the package of + // the Helm chart failed. + ChartPackageFailedReason string = "ChartPackageFailed" + + // ChartPackageSucceededReason represents the fact that the package of + // the Helm chart succeeded. + ChartPackageSucceededReason string = "ChartPackageSucceeded" ) -// HelmChartReady sets the given artifact and url on the HelmChart -// and resets the conditions to SourceCondition of type Ready with -// status true and the given reason and message. It returns the -// modified HelmChart. -func HelmChartReady(chart HelmChart, artifact Artifact, url, reason, message string) HelmChart { - chart.Status.Conditions = []SourceCondition{ - { - Type: ReadyCondition, - Status: corev1.ConditionTrue, - LastTransitionTime: metav1.Now(), - Reason: reason, - Message: message, - }, - } - chart.Status.URL = url - - if chart.Status.Artifact != nil { - if chart.Status.Artifact.Path != artifact.Path { - chart.Status.Artifact = &artifact - } - } else { - chart.Status.Artifact = &artifact - } - - return chart -} - -// HelmChartProgressing resets the conditions of the HelmChart -// to SourceCondition of type Ready with status unknown and -// progressing reason and message. It returns the modified HelmChart. +// HelmReleaseProgressing resets any failures and registers progress toward reconciling the given HelmRelease +// by setting the ReadyCondition to ConditionUnknown for ProgressingReason. func HelmChartProgressing(chart HelmChart) HelmChart { - chart.Status.Conditions = []SourceCondition{ - { - Type: ReadyCondition, - Status: corev1.ConditionUnknown, - LastTransitionTime: metav1.Now(), - Reason: ProgressingReason, - Message: "reconciliation in progress", - }, - } + chart.Status.URL = "" + chart.Status.Artifact = nil + chart.Status.Conditions = []SourceCondition{} + SetHelmChartCondition(&chart, ReadyCondition, corev1.ConditionUnknown, ProgressingReason, "reconciliation in progress") return chart } -// HelmChartNotReady resets the conditions of the HelmChart to -// SourceCondition of type Ready with status false and the given -// reason and message. It returns the modified HelmChart. +// SetHelmChartCondition sets the given condition with the given status, reason and message +// on the HelmChart. +func SetHelmChartCondition(chart *HelmChart, condition string, status corev1.ConditionStatus, reason, message string) { + chart.Status.Conditions = filterOutSourceCondition(chart.Status.Conditions, condition) + chart.Status.Conditions = append(chart.Status.Conditions, SourceCondition{ + Type: condition, + Status: status, + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: message, + }) +} + +// HelmChartReady sets the given artifact and url on the HelmChart +// and sets the ReadyCondition to True, with the given reason and +// message. It returns the modified HelmChart. +func HelmChartReady(chart HelmChart, artifact Artifact, url, reason, message string) HelmChart { + chart.Status.Artifact = &artifact + chart.Status.URL = url + SetHelmChartCondition(&chart, ReadyCondition, corev1.ConditionTrue, reason, message) + return chart +} + +// HelmChartNotReady sets the ReadyCondition on the given HelmChart +// to False, with the given reason and message. It returns the modified +// HelmChart. func HelmChartNotReady(chart HelmChart, reason, message string) HelmChart { - chart.Status.Conditions = []SourceCondition{ - { - Type: ReadyCondition, - Status: corev1.ConditionFalse, - LastTransitionTime: metav1.Now(), - Reason: reason, - Message: message, - }, - } + SetHelmChartCondition(&chart, ReadyCondition, corev1.ConditionFalse, reason, message) return chart } -// HelmChartReadyMessage returns the message of the SourceCondition -// of type Ready with status true if present, or an empty string. +// HelmChartReadyMessage returns the message of the ReadyCondition +// with status True, or an empty string. func HelmChartReadyMessage(chart HelmChart) string { for _, condition := range chart.Status.Conditions { if condition.Type == ReadyCondition && condition.Status == corev1.ConditionTrue { @@ -153,9 +159,10 @@ func (in *HelmChart) GetInterval() metav1.Duration { // +genclient:Namespaced // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="Name",type=string,JSONPath=`.spec.name` +// +kubebuilder:printcolumn:name="Chart",type=string,JSONPath=`.spec.chart` // +kubebuilder:printcolumn:name="Version",type=string,JSONPath=`.spec.version` -// +kubebuilder:printcolumn:name="Repository",type=string,JSONPath=`.spec.helmRepositoryRef.name` +// +kubebuilder:printcolumn:name="Source Kind",type=string,JSONPath=`.spec.sourceRef.kind` +// +kubebuilder:printcolumn:name="Source Name",type=string,JSONPath=`.spec.sourceRef.name` // +kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status",description="" // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].message",description="" // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="" diff --git a/api/v1alpha1/source.go b/api/v1alpha1/source.go index 6b13b49b..a4c3dfe8 100644 --- a/api/v1alpha1/source.go +++ b/api/v1alpha1/source.go @@ -17,3 +17,16 @@ const ( // reconciliation outside of the defined schedule. ReconcileAtAnnotation string = "fluxcd.io/reconcileAt" ) + +// filterOutSourceCondition returns a new SourceCondition slice without the +// SourceCondition of the given type. +func filterOutSourceCondition(conditions []SourceCondition, condition string) []SourceCondition { + var newConditions []SourceCondition + for _, c := range conditions { + if c.Type == condition { + continue + } + newConditions = append(newConditions, c) + } + return newConditions +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6cf06bb8..48003a1f 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -262,7 +262,7 @@ func (in *HelmChartList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HelmChartSpec) DeepCopyInto(out *HelmChartSpec) { *out = *in - out.HelmRepositoryRef = in.HelmRepositoryRef + out.SourceRef = in.SourceRef out.Interval = in.Interval } @@ -415,6 +415,21 @@ func (in *HelmRepositoryStatus) DeepCopy() *HelmRepositoryStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalHelmChartSourceReference) DeepCopyInto(out *LocalHelmChartSourceReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalHelmChartSourceReference. +func (in *LocalHelmChartSourceReference) DeepCopy() *LocalHelmChartSourceReference { + if in == nil { + return nil + } + out := new(LocalHelmChartSourceReference) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SourceCondition) DeepCopyInto(out *SourceCondition) { *out = *in diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml index 23c2156a..e66ef34d 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml @@ -17,14 +17,17 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - jsonPath: .spec.name - name: Name + - jsonPath: .spec.chart + name: Chart type: string - jsonPath: .spec.version name: Version type: string - - jsonPath: .spec.helmRepositoryRef.name - name: Repository + - jsonPath: .spec.sourceRef.kind + name: Source Kind + type: string + - jsonPath: .spec.sourceRef.name + name: Source Name type: string - jsonPath: .status.conditions[?(@.type=="Ready")].status name: Ready @@ -55,31 +58,41 @@ spec: spec: description: HelmChartSpec defines the desired state of a Helm chart. properties: - helmRepositoryRef: - description: The name of the HelmRepository the chart is available - at. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object + chart: + description: The name or path the Helm chart is available at in the + SourceRef. + type: string interval: - description: The interval at which to check the Helm repository for - updates. - type: string - name: - description: The name of the Helm chart, as made available by the - referenced Helm repository. + description: The interval at which to check the Source for updates. type: string + sourceRef: + description: The reference to the Source the chart is available at. + properties: + apiVersion: + description: APIVersion of the referent. + type: string + kind: + description: Kind of the referent, valid values are ('HelmRepository', + 'GitRepository'). + enum: + - HelmRepository + - GitRepository + type: string + name: + description: Name of the referent. + type: string + required: + - kind + - name + type: object version: - description: The chart version semver expression, defaults to latest - when omitted. + description: The chart version semver expression, ignored for charts + from GitRepository sources. Defaults to latest when omitted. type: string required: - - helmRepositoryRef + - chart - interval - - name + - sourceRef type: object status: description: HelmChartStatus defines the observed state of the HelmChart. diff --git a/config/samples/source_v1alpha1_helmchart_gitrepository.yaml b/config/samples/source_v1alpha1_helmchart_gitrepository.yaml new file mode 100644 index 00000000..5cb5feef --- /dev/null +++ b/config/samples/source_v1alpha1_helmchart_gitrepository.yaml @@ -0,0 +1,11 @@ +apiVersion: source.toolkit.fluxcd.io/v1alpha1 +kind: HelmChart +metadata: + name: helmchart-git-sample +spec: + chart: charts/podinfo + version: '^2.0.0' + sourceRef: + kind: GitRepository + name: gitrepository-sample + interval: 1m diff --git a/config/samples/source_v1alpha1_helmchart.yaml b/config/samples/source_v1alpha1_helmchart_helmrepository.yaml similarity index 76% rename from config/samples/source_v1alpha1_helmchart.yaml rename to config/samples/source_v1alpha1_helmchart_helmrepository.yaml index aa6be555..2aa32c54 100644 --- a/config/samples/source_v1alpha1_helmchart.yaml +++ b/config/samples/source_v1alpha1_helmchart_helmrepository.yaml @@ -3,8 +3,9 @@ kind: HelmChart metadata: name: helmchart-sample spec: - name: podinfo + chart: podinfo version: '>=2.0.0 <3.0.0' - helmRepositoryRef: + sourceRef: + kind: HelmRepository name: helmrepository-sample interval: 1m diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 47756d0c..844cf63c 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -21,13 +21,16 @@ import ( "fmt" "io/ioutil" "net/url" + "os" + "path" "strings" "time" "github.com/go-logr/logr" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chartutil" "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" kuberecorder "k8s.io/client-go/tools/record" @@ -37,6 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "github.com/fluxcd/pkg/recorder" + "github.com/fluxcd/pkg/untar" sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" "github.com/fluxcd/source-controller/internal/helm" @@ -99,13 +103,7 @@ func (r *HelmChartReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { } // set initial status - if reset, status := r.shouldResetStatus(chart); reset { - chart.Status = status - if err := r.Status().Update(ctx, &chart); err != nil { - log.Error(err, "unable to update status") - return ctrl.Result{Requeue: true}, err - } - } else { + if chart.Generation == 0 || chart.GetArtifact() != nil && !r.Storage.ArtifactExist(*chart.GetArtifact()) { chart = sourcev1.HelmChartProgressing(chart) if err := r.Status().Update(ctx, &chart); err != nil { log.Error(err, "unable to update status") @@ -118,19 +116,34 @@ func (r *HelmChartReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { log.Error(err, "unable to purge old artifacts") } - // get referenced chart repository - repository, err := r.getChartRepositoryWithArtifact(ctx, chart) - if err != nil { - chart = sourcev1.HelmChartNotReady(*chart.DeepCopy(), sourcev1.ChartPullFailedReason, err.Error()) - if err := r.Status().Update(ctx, &chart); err != nil { - log.Error(err, "unable to update status") + var reconciledChart sourcev1.HelmChart + var reconcileErr error + switch chart.Spec.SourceRef.Kind { + case sourcev1.HelmRepositoryKind: + repository, err := r.getChartRepositoryWithArtifact(ctx, chart) + if err != nil { + chart = sourcev1.HelmChartNotReady(*chart.DeepCopy(), sourcev1.ChartPullFailedReason, err.Error()) + if err := r.Status().Update(ctx, &chart); err != nil { + log.Error(err, "unable to update status") + } + return ctrl.Result{Requeue: true}, err } - return ctrl.Result{Requeue: true}, err + reconciledChart, reconcileErr = r.reconcileFromHelmRepository(ctx, repository, *chart.DeepCopy()) + case sourcev1.GitRepositoryKind: + repository, err := r.getGitRepositoryWithArtifact(ctx, chart) + if err != nil { + chart = sourcev1.HelmChartNotReady(*chart.DeepCopy(), sourcev1.ChartPullFailedReason, err.Error()) + if err := r.Status().Update(ctx, &chart); err != nil { + log.Error(err, "unable to update status") + } + return ctrl.Result{Requeue: true}, err + } + reconciledChart, reconcileErr = r.reconcileFromGitRepository(ctx, repository, *chart.DeepCopy()) + default: + err := fmt.Errorf("unable to reconcile unsupported source reference kind '%s'", chart.Spec.SourceRef.Kind) + return ctrl.Result{}, err } - // reconcile repository by downloading the chart tarball - reconciledChart, reconcileErr := r.reconcile(ctx, repository, *chart.DeepCopy()) - // update status with the reconciliation result if err := r.Status().Update(ctx, &reconciledChart); err != nil { log.Error(err, "unable to update status") @@ -172,8 +185,9 @@ func (r *HelmChartReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts Complete(r) } -func (r *HelmChartReconciler) reconcile(ctx context.Context, repository sourcev1.HelmRepository, chart sourcev1.HelmChart) (sourcev1.HelmChart, error) { - cv, err := helm.GetDownloadableChartVersionFromIndex(repository.Status.Artifact.Path, chart.Spec.Name, chart.Spec.Version) +func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context, + repository sourcev1.HelmRepository, chart sourcev1.HelmChart) (sourcev1.HelmChart, error) { + cv, err := helm.GetDownloadableChartVersionFromIndex(repository.Status.Artifact.Path, chart.Spec.Chart, chart.Spec.Version) if err != nil { return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err } @@ -251,14 +265,14 @@ func (r *HelmChartReconciler) reconcile(ctx context.Context, repository sourcev1 err = r.Storage.MkdirAll(artifact) if err != nil { err = fmt.Errorf("unable to create chart directory: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } // acquire lock unlock, err := r.Storage.Lock(artifact) if err != nil { err = fmt.Errorf("unable to acquire lock: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } defer unlock() @@ -266,10 +280,10 @@ func (r *HelmChartReconciler) reconcile(ctx context.Context, repository sourcev1 err = r.Storage.WriteFile(artifact, chartBytes) if err != nil { err = fmt.Errorf("unable to write chart file: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } - // update index symlink + // update symlink chartUrl, err := r.Storage.Symlink(artifact, fmt.Sprintf("%s-latest.tgz", cv.Name)) if err != nil { err = fmt.Errorf("storage error: %w", err) @@ -284,13 +298,13 @@ func (r *HelmChartReconciler) reconcile(ctx context.Context, repository sourcev1 // for the given chart. It returns an error if the HelmRepository could // not be retrieved or if does not have an artifact. func (r *HelmChartReconciler) getChartRepositoryWithArtifact(ctx context.Context, chart sourcev1.HelmChart) (sourcev1.HelmRepository, error) { - if chart.Spec.HelmRepositoryRef.Name == "" { + if chart.Spec.SourceRef.Name == "" { return sourcev1.HelmRepository{}, fmt.Errorf("no HelmRepository reference given") } name := types.NamespacedName{ Namespace: chart.GetNamespace(), - Name: chart.Spec.HelmRepositoryRef.Name, + Name: chart.Spec.SourceRef.Name, } var repository sourcev1.HelmRepository @@ -301,37 +315,117 @@ func (r *HelmChartReconciler) getChartRepositoryWithArtifact(ctx context.Context } if repository.Status.Artifact == nil { - err = fmt.Errorf("no repository index artifect found in HelmRepository '%s'", repository.Name) + err = fmt.Errorf("no repository index artifact found in HelmRepository '%s'", repository.Name) } return repository, err } -// shouldResetStatus returns a boolean indicating if the status of the -// given chart should be reset and a reset HelmChartStatus. -func (r *HelmChartReconciler) shouldResetStatus(chart sourcev1.HelmChart) (bool, sourcev1.HelmChartStatus) { - resetStatus := false - if chart.Status.Artifact != nil { - if !r.Storage.ArtifactExist(*chart.Status.Artifact) { - resetStatus = true - } +func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context, + repository sourcev1.GitRepository, chart sourcev1.HelmChart) (sourcev1.HelmChart, error) { + // create tmp dir + tmpDir, err := ioutil.TempDir("", fmt.Sprintf("%s-%s", chart.Namespace, chart.Name)) + if err != nil { + err = fmt.Errorf("tmp dir error: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + } + defer os.RemoveAll(tmpDir) + + // open file + f, err := os.Open(repository.GetArtifact().Path) + if err != nil { + err = fmt.Errorf("artifact open error: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } - // set initial status - if len(chart.Status.Conditions) == 0 { - resetStatus = true + // extract artifact files + if _, err = untar.Untar(f, tmpDir); err != nil { + err = fmt.Errorf("artifact untar error: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } - return resetStatus, sourcev1.HelmChartStatus{ - Conditions: []sourcev1.SourceCondition{ - { - Type: sourcev1.ReadyCondition, - Status: corev1.ConditionUnknown, - Reason: sourcev1.InitializingReason, - LastTransitionTime: metav1.Now(), - }, - }, + // ensure configured path is a chart directory + chartPath := path.Join(tmpDir, chart.Spec.Chart) + if _, err := chartutil.IsChartDir(chartPath); err != nil { + err = fmt.Errorf("chart path error: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } + + // read chart metadata + chartMetadata, err := chartutil.LoadChartfile(path.Join(chartPath, chartutil.ChartfileName)) + if err != nil { + err = fmt.Errorf("load chart metadata error: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + } + + // return early on unchanged chart version + if chart.Status.Artifact != nil && chartMetadata.Version == chart.Status.Artifact.Revision { + return chart, nil + } + + artifact := r.Storage.ArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(), + fmt.Sprintf("%s-%s.tgz", chartMetadata.Name, chartMetadata.Version), chartMetadata.Version) + + // create artifact dir + err = r.Storage.MkdirAll(artifact) + if err != nil { + err = fmt.Errorf("unable to create artifact directory: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + } + + // acquire lock + unlock, err := r.Storage.Lock(artifact) + if err != nil { + err = fmt.Errorf("unable to acquire lock: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + } + defer unlock() + + // package chart + pkg := action.NewPackage() + pkg.Destination = artifact.Path + _, err = pkg.Run(chartPath, nil) + if err != nil { + err = fmt.Errorf("chart package error: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err + } + + // update symlink + chartUrl, err := r.Storage.Symlink(artifact, fmt.Sprintf("%s-latest.tgz", chartMetadata.Name)) + if err != nil { + err = fmt.Errorf("storage error: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + } + + message := fmt.Sprintf("Fetched and packaged revision: %s", artifact.Revision) + return sourcev1.HelmChartReady(chart, artifact, chartUrl, sourcev1.ChartPackageSucceededReason, message), nil +} + +// getGitRepositoryWithArtifact attempts to get the GitRepository for the given +// chart. It returns an error if the GitRepository could not be retrieved or +// does not have an artifact. +func (r *HelmChartReconciler) getGitRepositoryWithArtifact(ctx context.Context, chart sourcev1.HelmChart) (sourcev1.GitRepository, error) { + if chart.Spec.SourceRef.Name == "" { + return sourcev1.GitRepository{}, fmt.Errorf("no GitRepository reference given") + } + + name := types.NamespacedName{ + Namespace: chart.GetNamespace(), + Name: chart.Spec.SourceRef.Name, + } + + var repository sourcev1.GitRepository + err := r.Client.Get(ctx, name, &repository) + if err != nil { + err = fmt.Errorf("failed to get GitRepository '%s': %w", name, err) + return repository, err + } + + if repository.Status.Artifact == nil { + err = fmt.Errorf("no artifact found for GitRepository '%s'", repository.Name) + } + + return repository, err } // gc performs a garbage collection on all but current artifacts of diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go index 67c8c0ad..2ca476a2 100644 --- a/controllers/helmchart_controller_test.go +++ b/controllers/helmchart_controller_test.go @@ -18,19 +18,31 @@ package controllers import ( "context" + "fmt" + "io/ioutil" "net/http" + "net/url" "os" "path" + "path/filepath" "strings" "time" + "github.com/fluxcd/pkg/gittestserver" + "github.com/fluxcd/pkg/helmtestserver" + "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage/memory" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chartutil" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - - "github.com/fluxcd/pkg/helmtestserver" + "sigs.k8s.io/yaml" sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" ) @@ -44,7 +56,7 @@ var _ = Describe("HelmChartReconciler", func() { pullInterval = time.Second * 3 ) - Context("HelmChart", func() { + Context("HelmChart from HelmRepository", func() { var ( namespace *corev1.Namespace helmServer *helmtestserver.HelmServer @@ -53,7 +65,7 @@ var _ = Describe("HelmChartReconciler", func() { BeforeEach(func() { namespace = &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "helm-chart-test" + randStringRunes(5)}, + ObjectMeta: metav1.ObjectMeta{Name: "helm-chart-test-" + randStringRunes(5)}, } err = k8sClient.Create(context.Background(), namespace) Expect(err).NotTo(HaveOccurred(), "failed to create test namespace") @@ -100,10 +112,13 @@ var _ = Describe("HelmChartReconciler", func() { Namespace: key.Namespace, }, Spec: sourcev1.HelmChartSpec{ - Name: "helmchart", - Version: "", - HelmRepositoryRef: corev1.LocalObjectReference{Name: repositoryKey.Name}, - Interval: metav1.Duration{Duration: pullInterval}, + Chart: "helmchart", + Version: "", + SourceRef: sourcev1.LocalHelmChartSourceReference{ + Kind: sourcev1.HelmRepositoryKind, + Name: repositoryKey.Name, + }, + Interval: metav1.Duration{Duration: pullInterval}, }, } Expect(k8sClient.Create(context.Background(), created)).Should(Succeed()) @@ -131,7 +146,7 @@ var _ = Describe("HelmChartReconciler", func() { By("Expecting missing HelmRepository error") updated := &sourcev1.HelmChart{} Expect(k8sClient.Get(context.Background(), key, updated)).Should(Succeed()) - updated.Spec.HelmRepositoryRef.Name = "invalid" + updated.Spec.SourceRef.Name = "invalid" Expect(k8sClient.Update(context.Background(), updated)).Should(Succeed()) Eventually(func() bool { _ = k8sClient.Get(context.Background(), key, updated) @@ -156,7 +171,7 @@ var _ = Describe("HelmChartReconciler", func() { Eventually(func() error { c := &sourcev1.HelmChart{} return k8sClient.Get(context.Background(), key, c) - }).ShouldNot(Succeed()) + }, timeout, interval).ShouldNot(Succeed()) exists := func(path string) bool { // wait for tmp sync on macOS @@ -181,7 +196,7 @@ var _ = Describe("HelmChartReconciler", func() { Name: "helmrepository-sample-" + randStringRunes(5), Namespace: namespace.Name, } - Expect(k8sClient.Create(context.Background(), &sourcev1.HelmRepository{ + repository := &sourcev1.HelmRepository{ ObjectMeta: metav1.ObjectMeta{ Name: repositoryKey.Name, Namespace: repositoryKey.Namespace, @@ -190,7 +205,9 @@ var _ = Describe("HelmChartReconciler", func() { URL: helmServer.URL(), Interval: metav1.Duration{Duration: 1 * time.Hour}, }, - })).Should(Succeed()) + } + Expect(k8sClient.Create(context.Background(), repository)).Should(Succeed()) + defer k8sClient.Delete(context.Background(), repository) key := types.NamespacedName{ Name: "helmchart-sample-" + randStringRunes(5), @@ -202,12 +219,17 @@ var _ = Describe("HelmChartReconciler", func() { Namespace: key.Namespace, }, Spec: sourcev1.HelmChartSpec{ - Name: "helmchart", - HelmRepositoryRef: corev1.LocalObjectReference{Name: repositoryKey.Name}, - Interval: metav1.Duration{Duration: 1 * time.Hour}, + Chart: "helmchart", + Version: "*", + SourceRef: sourcev1.LocalHelmChartSourceReference{ + Kind: sourcev1.HelmRepositoryKind, + Name: repositoryKey.Name, + }, + Interval: metav1.Duration{Duration: 1 * time.Hour}, }, } Expect(k8sClient.Create(context.Background(), chart)).Should(Succeed()) + defer k8sClient.Delete(context.Background(), chart) Eventually(func() string { _ = k8sClient.Get(context.Background(), key, chart) @@ -293,6 +315,7 @@ var _ = Describe("HelmChartReconciler", func() { }, } Expect(k8sClient.Create(context.Background(), repository)).Should(Succeed()) + defer k8sClient.Delete(context.Background(), repository) key := types.NamespacedName{ Name: "helmchart-sample-" + randStringRunes(5), @@ -304,13 +327,17 @@ var _ = Describe("HelmChartReconciler", func() { Namespace: key.Namespace, }, Spec: sourcev1.HelmChartSpec{ - Name: "helmchart", - Version: "*", - HelmRepositoryRef: corev1.LocalObjectReference{Name: repositoryKey.Name}, - Interval: metav1.Duration{Duration: pullInterval}, + Chart: "helmchart", + Version: "*", + SourceRef: sourcev1.LocalHelmChartSourceReference{ + Kind: sourcev1.HelmRepositoryKind, + Name: repositoryKey.Name, + }, + Interval: metav1.Duration{Duration: pullInterval}, }, } Expect(k8sClient.Create(context.Background(), chart)).Should(Succeed()) + defer k8sClient.Delete(context.Background(), chart) By("Expecting artifact") Expect(k8sClient.Update(context.Background(), secret)).Should(Succeed()) @@ -371,4 +398,185 @@ var _ = Describe("HelmChartReconciler", func() { Expect(got.Status.Artifact).ShouldNot(BeNil()) }) }) + + Context("HelmChart from GitRepository", func() { + var ( + namespace *corev1.Namespace + gitServer *gittestserver.GitServer + err error + ) + + BeforeEach(func() { + namespace = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "test-git-repository-" + randStringRunes(5)}, + } + err = k8sClient.Create(context.Background(), namespace) + Expect(err).NotTo(HaveOccurred(), "failed to create test namespace") + + gitServer, err = gittestserver.NewTempGitServer() + Expect(err).NotTo(HaveOccurred()) + gitServer.AutoCreate() + Expect(gitServer.StartHTTP()).To(Succeed()) + }) + + AfterEach(func() { + gitServer.StopHTTP() + os.RemoveAll(gitServer.Root()) + + err = k8sClient.Delete(context.Background(), namespace) + Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace") + }) + + It("Creates artifacts for", func() { + fs := memfs.New() + gitrepo, err := git.Init(memory.NewStorage(), fs) + Expect(err).NotTo(HaveOccurred()) + + wt, err := gitrepo.Worktree() + Expect(err).NotTo(HaveOccurred()) + + u, err := url.Parse(gitServer.HTTPAddress()) + Expect(err).NotTo(HaveOccurred()) + u.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5))) + + _, err = gitrepo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{u.String()}, + }) + Expect(err).NotTo(HaveOccurred()) + + chartDir := "testdata/helmchart" + Expect(filepath.Walk(chartDir, func(p string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + switch { + case fi.Mode().IsDir(): + return fs.MkdirAll(p, os.ModeDir) + case !fi.Mode().IsRegular(): + return nil + } + + b, err := ioutil.ReadFile(p) + if err != nil { + return err + } + + ff, err := fs.Create(p) + if err != nil { + return err + } + if _, err := ff.Write(b); err != nil { + return err + } + _ = ff.Close() + _, err = wt.Add(p) + + return err + })).To(Succeed()) + + _, err = wt.Commit("Helm chart", &git.CommitOptions{Author: &object.Signature{ + Name: "John Doe", + Email: "john@example.com", + When: time.Now(), + }}) + Expect(err).NotTo(HaveOccurred()) + + err = gitrepo.Push(&git.PushOptions{}) + Expect(err).NotTo(HaveOccurred()) + + repositoryKey := types.NamespacedName{ + Name: fmt.Sprintf("git-repository-sample-%s", randStringRunes(5)), + Namespace: namespace.Name, + } + repository := &sourcev1.GitRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: repositoryKey.Name, + Namespace: repositoryKey.Namespace, + }, + Spec: sourcev1.GitRepositorySpec{ + URL: u.String(), + Interval: metav1.Duration{Duration: indexInterval}, + }, + } + Expect(k8sClient.Create(context.Background(), repository)).Should(Succeed()) + defer k8sClient.Delete(context.Background(), repository) + + key := types.NamespacedName{ + Name: "helmchart-sample-" + randStringRunes(5), + Namespace: namespace.Name, + } + chart := &sourcev1.HelmChart{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + Spec: sourcev1.HelmChartSpec{ + Chart: "testdata/helmchart", + Version: "*", + SourceRef: sourcev1.LocalHelmChartSourceReference{ + Kind: sourcev1.GitRepositoryKind, + Name: repositoryKey.Name, + }, + Interval: metav1.Duration{Duration: pullInterval}, + }, + } + Expect(k8sClient.Create(context.Background(), chart)).Should(Succeed()) + defer k8sClient.Delete(context.Background(), chart) + + By("Expecting artifact") + got := &sourcev1.HelmChart{} + Eventually(func() bool { + _ = k8sClient.Get(context.Background(), key, got) + return got.Status.Artifact != nil && + storage.ArtifactExist(*got.Status.Artifact) + }, timeout, interval).Should(BeTrue()) + + By("Committing a new version in the chart metadata") + f, err := fs.OpenFile(fs.Join(chartDir, chartutil.ChartfileName), os.O_RDWR, os.FileMode(0600)) + Expect(err).NotTo(HaveOccurred()) + + b := make([]byte, 1024) + n, err := f.Read(b) + Expect(err).NotTo(HaveOccurred()) + b = b[0:n] + + y := new(helmchart.Metadata) + err = yaml.Unmarshal(b, y) + Expect(err).NotTo(HaveOccurred()) + + y.Version = "0.2.0" + b, err = yaml.Marshal(y) + Expect(err).NotTo(HaveOccurred()) + + _, err = f.Write(b) + Expect(err).NotTo(HaveOccurred()) + + err = f.Close() + Expect(err).NotTo(HaveOccurred()) + + _, err = wt.Commit("Chart version bump", &git.CommitOptions{ + Author: &object.Signature{ + Name: "John Doe", + Email: "john@example.com", + When: time.Now(), + }, + All: true, + }) + Expect(err).NotTo(HaveOccurred()) + + err = gitrepo.Push(&git.PushOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Expecting new artifact revision and GC") + Eventually(func() bool { + now := &sourcev1.HelmChart{} + _ = k8sClient.Get(context.Background(), key, now) + // Test revision change and garbage collection + return now.Status.Artifact.Revision != got.Status.Artifact.Revision && + !storage.ArtifactExist(*got.Status.Artifact) + }, timeout, interval).Should(BeTrue()) + }) + }) }) diff --git a/controllers/storage.go b/controllers/storage.go index 748faf07..a0a51d29 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fluxcd/pkg/lockedfile" + sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" ) @@ -129,7 +130,6 @@ func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool { // Archive creates a tar.gz to the artifact path from the given dir excluding any VCS specific // files and directories, or any of the excludes defined in the excludeFiles. -// Returns a modified sourcev1.Artifact and any error. func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.GitRepositorySpec) error { if _, err := os.Stat(dir); err != nil { return err @@ -250,7 +250,7 @@ func (s *Storage) Lock(artifact sourcev1.Artifact) (unlock func(), err error) { } func getPatterns(reader io.Reader, path []string) []gitignore.Pattern { - ps := []gitignore.Pattern{} + var ps []gitignore.Pattern scanner := bufio.NewScanner(reader) for scanner.Scan() { diff --git a/controllers/suite_test.go b/controllers/suite_test.go index cf03612b..74912c9d 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -163,15 +163,15 @@ func init() { } func loadExampleKeys() (err error) { - examplePublicKey, err = ioutil.ReadFile(filepath.Join("testdata/certs/server.pem")) + examplePublicKey, err = ioutil.ReadFile("testdata/certs/server.pem") if err != nil { return err } - examplePrivateKey, err = ioutil.ReadFile(filepath.Join("testdata/certs/server-key.pem")) + examplePrivateKey, err = ioutil.ReadFile("testdata/certs/server-key.pem") if err != nil { return err } - exampleCA, err = ioutil.ReadFile(filepath.Join("testdata/certs/ca.pem")) + exampleCA, err = ioutil.ReadFile("testdata/certs/ca.pem") return err } diff --git a/docs/api/source.md b/docs/api/source.md index 6674e961..b32eea5f 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -250,14 +250,13 @@ HelmChartSpec @@ -269,21 +268,21 @@ string @@ -296,7 +295,7 @@ Kubernetes meta/v1.Duration
-name
+chart
string
-

The name of the Helm chart, as made available by the referenced -Helm repository.

+

The name or path the Helm chart is available at in the SourceRef.

(Optional) -

The chart version semver expression, defaults to latest when -omitted.

+

The chart version semver expression, ignored for charts from GitRepository +sources. Defaults to latest when omitted.

-helmRepositoryRef
+sourceRef
- -Kubernetes core/v1.LocalObjectReference + +LocalHelmChartSourceReference
-

The name of the HelmRepository the chart is available at.

+

The reference to the Source the chart is available at.

-

The interval at which to check the Helm repository for updates.

+

The interval at which to check the Source for updates.

@@ -839,14 +838,13 @@ Kubernetes core/v1.LocalObjectReference -name
+chart
string -

The name of the Helm chart, as made available by the referenced -Helm repository.

+

The name or path the Helm chart is available at in the SourceRef.

@@ -858,21 +856,21 @@ string (Optional) -

The chart version semver expression, defaults to latest when -omitted.

+

The chart version semver expression, ignored for charts from GitRepository +sources. Defaults to latest when omitted.

-helmRepositoryRef
+sourceRef
- -Kubernetes core/v1.LocalObjectReference + +LocalHelmChartSourceReference -

The name of the HelmRepository the chart is available at.

+

The reference to the Source the chart is available at.

@@ -885,7 +883,7 @@ Kubernetes meta/v1.Duration -

The interval at which to check the Helm repository for updates.

+

The interval at which to check the Source for updates.

@@ -1090,6 +1088,62 @@ Artifact +

LocalHelmChartSourceReference +

+

+(Appears on: +HelmChartSpec) +

+

LocalHelmChartSourceReference contains enough information to let you locate the +typed referenced object at namespace level.

+
+
+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+apiVersion
+ +string + +
+(Optional) +

APIVersion of the referent.

+
+kind
+ +string + +
+

Kind of the referent, valid values are (‘HelmRepository’, ‘GitRepository’).

+
+name
+ +string + +
+

Name of the referent.

+
+
+

Source

Source interface must be supported by all API types.

diff --git a/docs/spec/v1alpha1/helmcharts.md b/docs/spec/v1alpha1/helmcharts.md index 2608f651..5b4527c2 100644 --- a/docs/spec/v1alpha1/helmcharts.md +++ b/docs/spec/v1alpha1/helmcharts.md @@ -11,27 +11,46 @@ Helm chart: ```go // HelmChartSpec defines the desired state of a Helm chart source. type HelmChartSpec struct { - // The name of the Helm chart, as made available by the referenced - // Helm repository. + // The name or path the Helm chart is available at in the SourceRef. // +required - Name string `json:"name"` + Chart string `json:"chart"` - // The chart version semver expression, defaults to latest when - // omitted. + // The chart version semver expression, ignored for charts from GitRepository + // sources. Defaults to latest when omitted. // +optional Version string `json:"version,omitempty"` - // The name of the HelmRepository the chart is available at. + // The reference to the Source the chart is available at. // +required - HelmRepositoryRef v1.LocalObjectReference `json:"helmRepositoryRef"` + SourceRef LocalHelmChartSourceReference `json:"sourceRef"` - // The interval at which to check the referenced HelmRepository index - // for updates. + // The interval at which to check the Source for updates. // +required Interval metav1.Duration `json:"interval"` } ``` +### Reference types + +```go +// LocalHelmChartSourceReference contains enough information to let you locate the +// typed referenced object at namespace level. +type LocalHelmChartSourceReference struct { + // APIVersion of the referent. + // +optional + APIVersion string `json:"apiVersion,omitempty"` + + // Kind of the referent, valid values are ('HelmRepository', 'GitRepository'). + // +kubebuilder:validation:Enum=HelmRepository;GitRepository + // +required + Kind string `json:"kind"` + + // Name of the referent. + // +required + Name string `json:"name"` +} +``` + ### Status ```go @@ -61,6 +80,14 @@ const ( // ChartPullSucceededReason represents the fact that the pull of // the given Helm chart succeeded. ChartPullSucceededReason string = "ChartPullSucceeded" + + // ChartPackageFailedReason represent the fact that the package of + // the Helm chart failed. + ChartPackageFailedReason string = "ChartPackageFailed" + + // ChartPackageSucceededReason represents the fact that the package of + // the Helm chart succeeded. + ChartPackageSucceededReason string = "ChartPackageSucceeded" ) ``` diff --git a/go.mod b/go.mod index 7f8375df..370e7b2d 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/fluxcd/pkg/logger v0.0.1 github.com/fluxcd/pkg/recorder v0.0.6 github.com/fluxcd/pkg/ssh v0.0.5 + github.com/fluxcd/pkg/untar v0.0.5 github.com/fluxcd/source-controller/api v0.0.10 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.1.0 diff --git a/go.sum b/go.sum index 714e5275..5b0857cc 100644 --- a/go.sum +++ b/go.sum @@ -216,6 +216,8 @@ github.com/fluxcd/pkg/ssh v0.0.5 h1:rnbFZ7voy2JBlUfMbfyqArX2FYaLNpDhccGFC3qW83A= github.com/fluxcd/pkg/ssh v0.0.5/go.mod h1:7jXPdXZpc0ttMNz2kD9QuMi3RNn/e0DOFbj0Tij/+Hs= github.com/fluxcd/pkg/testserver v0.0.2 h1:SoaMtO9cE5p/wl2zkGudzflnEHd9mk68CGjZOo7w0Uk= github.com/fluxcd/pkg/testserver v0.0.2/go.mod h1:pgUZTh9aQ44FSTQo+5NFlh7YMbUfdz1B80DalW7k96Y= +github.com/fluxcd/pkg/untar v0.0.5 h1:UGI3Ch1UIEIaqQvMicmImL1s9npQa64DJ/ozqHKB7gk= +github.com/fluxcd/pkg/untar v0.0.5/go.mod h1:O6V9+rtl8c1mHBafgqFlJN6zkF1HS5SSYn7RpQJ/nfw= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=