fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

controllers: RFC-0005 fmt for HelmRepository rev 

This includes changes to the `ChartRepository`, to allow calculating
the revision and digest and tidy things.

In addition, the responsibility of caching the `IndexFile` has been
moved to the reconcilers. As this allowed to remove a lot of
complexities within the `ChartRepository`, and prevented passing on
the cache in general.

Change `HelmRepository`'s Revision to digest

Signed-off-by: Hidde Beydals <hello@hidde.co>
This commit is contained in:
Hidde Beydals 2023-01-19 16:40:10 +00:00
parent f00aeae09a
commit 0aaeeee5e9
7 changed files with 885 additions and 704 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
controllers/helmchart_controller.go
View File

@ -21,8 +21,6 @@ import (
"crypto/tls" "crypto/tls"
"errors" "errors"
"fmt" "fmt"
"github.com/fluxcd/pkg/git"
"github.com/opencontainers/go-digest"
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
@ -30,12 +28,12 @@ import (
"strings" "strings"
"time" "time"
eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1"
soci "github.com/fluxcd/source-controller/internal/oci"
"github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/opencontainers/go-digest"
helmgetter "helm.sh/helm/v3/pkg/getter" helmgetter "helm.sh/helm/v3/pkg/getter"
helmreg "helm.sh/helm/v3/pkg/registry" helmreg "helm.sh/helm/v3/pkg/registry"
helmrepo "helm.sh/helm/v3/pkg/repo"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors" apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -54,7 +52,9 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source" "sigs.k8s.io/controller-runtime/pkg/source"
eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1"
"github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/oci" "github.com/fluxcd/pkg/oci"
"github.com/fluxcd/pkg/runtime/conditions" "github.com/fluxcd/pkg/runtime/conditions"
helper "github.com/fluxcd/pkg/runtime/controller" helper "github.com/fluxcd/pkg/runtime/controller"
@ -70,6 +70,7 @@ import (
"github.com/fluxcd/source-controller/internal/helm/getter" "github.com/fluxcd/source-controller/internal/helm/getter"
"github.com/fluxcd/source-controller/internal/helm/registry" "github.com/fluxcd/source-controller/internal/helm/registry"
"github.com/fluxcd/source-controller/internal/helm/repository" "github.com/fluxcd/source-controller/internal/helm/repository"
soci "github.com/fluxcd/source-controller/internal/oci"
sreconcile "github.com/fluxcd/source-controller/internal/reconcile" sreconcile "github.com/fluxcd/source-controller/internal/reconcile"
"github.com/fluxcd/source-controller/internal/reconcile/summarize" "github.com/fluxcd/source-controller/internal/reconcile/summarize"
"github.com/fluxcd/source-controller/internal/util" "github.com/fluxcd/source-controller/internal/util"
@ -527,7 +528,7 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
} }
// Build client options from secret // Build client options from secret
opts, tls, err := r.clientOptionsFromSecret(secret, normalizedURL) opts, tlsCfg, err := r.clientOptionsFromSecret(secret, normalizedURL)
if err != nil { if err != nil {
e := &serror.Event{ e := &serror.Event{
Err: err, Err: err,
@ -538,7 +539,7 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
return sreconcile.ResultEmpty, e return sreconcile.ResultEmpty, e
} }
clientOpts = append(clientOpts, opts...) clientOpts = append(clientOpts, opts...)
tlsConfig = tls tlsConfig = tlsCfg
// Build registryClient options from secret // Build registryClient options from secret
keychain, err = registry.LoginOptionFromSecret(normalizedURL, *secret) keychain, err = registry.LoginOptionFromSecret(normalizedURL, *secret)
@ -651,35 +652,38 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
} }
} }
default: default:
httpChartRepo, err := repository.NewChartRepository(normalizedURL, r.Storage.LocalPath(*repo.GetArtifact()), r.Getters, tlsConfig, clientOpts, httpChartRepo, err := repository.NewChartRepository(normalizedURL, r.Storage.LocalPath(*repo.GetArtifact()), r.Getters, tlsConfig, clientOpts...)
repository.WithMemoryCache(r.Storage.LocalPath(*repo.GetArtifact()), r.Cache, r.TTL, func(event string) {
r.IncCacheEvents(event, obj.Name, obj.Namespace)
}))
if err != nil { if err != nil {
return chartRepoConfigErrorReturn(err, obj) return chartRepoConfigErrorReturn(err, obj)
} }
chartRepo = httpChartRepo
defer func() {
if httpChartRepo == nil {
return
}
// Cache the index if it was successfully retrieved
// and the chart was successfully built
if r.Cache != nil && httpChartRepo.Index != nil {
// The cache key have to be safe in multi-tenancy environments,
// as otherwise it could be used as a vector to bypass the helm repository's authentication.
// Using r.Storage.LocalPath(*repo.GetArtifact() is safe as the path is in the format /<helm-repository-name>/<chart-name>/<filename>.
err := httpChartRepo.CacheIndexInMemory()
if err != nil {
r.eventLogf(ctx, obj, eventv1.EventTypeTrace, sourcev1.CacheOperationFailedReason, "failed to cache index: %s", err)
}
}
// Delete the index reference // NB: this needs to be deferred first, as otherwise the Index will disappear
if httpChartRepo.Index != nil { // before we had a chance to cache it.
httpChartRepo.Unload() defer func() {
if err := httpChartRepo.Clear(); err != nil {
ctrl.LoggerFrom(ctx).Error(err, "failed to clear Helm repository index")
} }
}() }()
// Attempt to load the index from the cache.
if r.Cache != nil {
if index, ok := r.Cache.Get(httpChartRepo.Path); ok {
r.IncCacheEvents(cache.CacheEventTypeHit, repo.Name, repo.Namespace)
r.Cache.SetExpiration(httpChartRepo.Path, r.TTL)
httpChartRepo.Index = index.(*helmrepo.IndexFile)
} else {
r.IncCacheEvents(cache.CacheEventTypeMiss, repo.Name, repo.Namespace)
defer func() {
// If we succeed in loading the index, cache it.
if httpChartRepo.Index != nil {
if err = r.Cache.Set(httpChartRepo.Path, httpChartRepo.Index, r.TTL); err != nil {
r.eventLogf(ctx, obj, eventv1.EventTypeTrace, sourcev1.CacheOperationFailedReason, "failed to cache index: %s", err)
}
}
}()
}
}
chartRepo = httpChartRepo
} }
// Construct the chart builder with scoped configuration // Construct the chart builder with scoped configuration
@ -845,7 +849,7 @@ func (r *HelmChartReconciler) buildFromTarballArtifact(ctx context.Context, obj
// early. // early.
// On a successful archive, the Artifact in the Status of the object is set, // On a successful archive, the Artifact in the Status of the object is set,
// and the symlink in the Storage is updated to its path. // and the symlink in the Storage is updated to its path.
func (r *HelmChartReconciler) reconcileArtifact(ctx context.Context, sp *patch.SerialPatcher, obj *sourcev1.HelmChart, b *chart.Build) (sreconcile.Result, error) { func (r *HelmChartReconciler) reconcileArtifact(ctx context.Context, _ *patch.SerialPatcher, obj *sourcev1.HelmChart, b *chart.Build) (sreconcile.Result, error) {
// Without a complete chart build, there is little to reconcile // Without a complete chart build, there is little to reconcile
if !b.Complete() { if !b.Complete() {
return sreconcile.ResultRequeue, nil return sreconcile.ResultRequeue, nil
@ -1016,14 +1020,15 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont
authenticator authn.Authenticator authenticator authn.Authenticator
keychain authn.Keychain keychain authn.Keychain
) )
normalizedURL := repository.NormalizeURL(url) normalizedURL := repository.NormalizeURL(url)
repo, err := r.resolveDependencyRepository(ctx, url, namespace) obj, err := r.resolveDependencyRepository(ctx, url, namespace)
if err != nil { if err != nil {
// Return Kubernetes client errors, but ignore others // Return Kubernetes client errors, but ignore others
if apierrs.ReasonForError(err) != metav1.StatusReasonUnknown { if apierrs.ReasonForError(err) != metav1.StatusReasonUnknown {
return nil, err return nil, err
} }
repo = &sourcev1.HelmRepository{ obj = &sourcev1.HelmRepository{
Spec: sourcev1.HelmRepositorySpec{ Spec: sourcev1.HelmRepositorySpec{
URL: url, URL: url,
Timeout: &metav1.Duration{Duration: 60 * time.Second}, Timeout: &metav1.Duration{Duration: 60 * time.Second},
@ -1032,37 +1037,37 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont
} }
// Used to login with the repository declared provider // Used to login with the repository declared provider
ctxTimeout, cancel := context.WithTimeout(ctx, repo.Spec.Timeout.Duration) ctxTimeout, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
defer cancel() defer cancel()
clientOpts := []helmgetter.Option{ clientOpts := []helmgetter.Option{
helmgetter.WithURL(normalizedURL), helmgetter.WithURL(normalizedURL),
helmgetter.WithTimeout(repo.Spec.Timeout.Duration), helmgetter.WithTimeout(obj.Spec.Timeout.Duration),
helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), helmgetter.WithPassCredentialsAll(obj.Spec.PassCredentials),
} }
if secret, err := r.getHelmRepositorySecret(ctx, repo); secret != nil || err != nil { if secret, err := r.getHelmRepositorySecret(ctx, obj); secret != nil || err != nil {
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Build client options from secret // Build client options from secret
opts, tls, err := r.clientOptionsFromSecret(secret, normalizedURL) opts, tlsCfg, err := r.clientOptionsFromSecret(secret, normalizedURL)
if err != nil { if err != nil {
return nil, err return nil, err
} }
clientOpts = append(clientOpts, opts...) clientOpts = append(clientOpts, opts...)
tlsConfig = tls tlsConfig = tlsCfg
// Build registryClient options from secret // Build registryClient options from secret
keychain, err = registry.LoginOptionFromSecret(normalizedURL, *secret) keychain, err = registry.LoginOptionFromSecret(normalizedURL, *secret)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create login options for HelmRepository '%s': %w", repo.Name, err) return nil, fmt.Errorf("failed to create login options for HelmRepository '%s': %w", obj.Name, err)
} }
} else if repo.Spec.Provider != sourcev1.GenericOCIProvider && repo.Spec.Type == sourcev1.HelmRepositoryTypeOCI { } else if obj.Spec.Provider != sourcev1.GenericOCIProvider && obj.Spec.Type == sourcev1.HelmRepositoryTypeOCI {
auth, authErr := oidcAuth(ctxTimeout, repo.Spec.URL, repo.Spec.Provider) auth, authErr := oidcAuth(ctxTimeout, obj.Spec.URL, obj.Spec.Provider)
if authErr != nil && !errors.Is(authErr, oci.ErrUnconfiguredProvider) { if authErr != nil && !errors.Is(authErr, oci.ErrUnconfiguredProvider) {
return nil, fmt.Errorf("failed to get credential from %s: %w", repo.Spec.Provider, authErr) return nil, fmt.Errorf("failed to get credential from %s: %w", obj.Spec.Provider, authErr)
} }
if auth != nil { if auth != nil {
authenticator = auth authenticator = auth
@ -1078,7 +1083,7 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont
if helmreg.IsOCI(normalizedURL) { if helmreg.IsOCI(normalizedURL) {
registryClient, credentialsFile, err := r.RegistryClientGenerator(loginOpt != nil) registryClient, credentialsFile, err := r.RegistryClientGenerator(loginOpt != nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to create registry client for HelmRepository '%s': %w", repo.Name, err) return nil, fmt.Errorf("failed to create registry client for HelmRepository '%s': %w", obj.Name, err)
} }
var errs []error var errs []error
@ -1089,7 +1094,7 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont
repository.WithOCIRegistryClient(registryClient), repository.WithOCIRegistryClient(registryClient),
repository.WithCredentialsFile(credentialsFile)) repository.WithCredentialsFile(credentialsFile))
if err != nil { if err != nil {
errs = append(errs, fmt.Errorf("failed to create OCI chart repository for HelmRepository '%s': %w", repo.Name, err)) errs = append(errs, fmt.Errorf("failed to create OCI chart repository for HelmRepository '%s': %w", obj.Name, err))
// clean up the credentialsFile // clean up the credentialsFile
if credentialsFile != "" { if credentialsFile != "" {
if err := os.Remove(credentialsFile); err != nil { if err := os.Remove(credentialsFile); err != nil {
@ -1104,7 +1109,7 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont
if loginOpt != nil { if loginOpt != nil {
err = ociChartRepo.Login(loginOpt) err = ociChartRepo.Login(loginOpt)
if err != nil { if err != nil {
errs = append(errs, fmt.Errorf("failed to login to OCI chart repository for HelmRepository '%s': %w", repo.Name, err)) errs = append(errs, fmt.Errorf("failed to login to OCI chart repository for HelmRepository '%s': %w", obj.Name, err))
// clean up the credentialsFile // clean up the credentialsFile
errs = append(errs, ociChartRepo.Clear()) errs = append(errs, ociChartRepo.Clear())
return nil, kerrors.NewAggregate(errs) return nil, kerrors.NewAggregate(errs)
@ -1113,19 +1118,28 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont
chartRepo = ociChartRepo chartRepo = ociChartRepo
} else { } else {
httpChartRepo, err := repository.NewChartRepository(normalizedURL, "", r.Getters, tlsConfig, clientOpts) httpChartRepo, err := repository.NewChartRepository(normalizedURL, "", r.Getters, tlsConfig, clientOpts...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Ensure that the cache key is the same as the artifact path if obj.Status.Artifact != nil {
// otherwise don't enable caching. We don't want to cache indexes // Attempt to load the index from the cache.
// for repositories that are not reconciled by the source controller. httpChartRepo.Path = r.Storage.LocalPath(*obj.GetArtifact())
if repo.Status.Artifact != nil { if r.Cache != nil {
httpChartRepo.CachePath = r.Storage.LocalPath(*repo.GetArtifact()) if index, ok := r.Cache.Get(httpChartRepo.Path); ok {
httpChartRepo.SetMemCache(r.Storage.LocalPath(*repo.GetArtifact()), r.Cache, r.TTL, func(event string) { r.IncCacheEvents(cache.CacheEventTypeHit, name, namespace)
r.IncCacheEvents(event, name, namespace) r.Cache.SetExpiration(httpChartRepo.Path, r.TTL)
})
httpChartRepo.Index = index.(*helmrepo.IndexFile)
} else {
r.IncCacheEvents(cache.CacheEventTypeMiss, name, namespace)
if err := httpChartRepo.LoadFromPath(); err != nil {
return nil, err
}
r.Cache.Set(httpChartRepo.Path, httpChartRepo.Index, r.TTL)
}
}
} }
chartRepo = httpChartRepo chartRepo = httpChartRepo

119
controllers/helmrepository_controller.go
View File

@ -21,10 +21,12 @@ import (
"crypto/tls" "crypto/tls"
"errors" "errors"
"fmt" "fmt"
"github.com/fluxcd/source-controller/internal/digest"
"net/url" "net/url"
"time" "time"
"github.com/docker/go-units" "github.com/docker/go-units"
digestlib "github.com/opencontainers/go-digest"
helmgetter "helm.sh/helm/v3/pkg/getter" helmgetter "helm.sh/helm/v3/pkg/getter"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
@ -277,13 +279,13 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, sp *patch.Seri
res = sreconcile.LowestRequeuingResult(res, recResult) res = sreconcile.LowestRequeuingResult(res, recResult)
} }
r.notify(ctx, oldObj, obj, chartRepo, res, resErr) r.notify(ctx, oldObj, obj, &chartRepo, res, resErr)
return res, resErr return res, resErr
} }
// notify emits notification related to the reconciliation. // notify emits notification related to the reconciliation.
func (r *HelmRepositoryReconciler) notify(ctx context.Context, oldObj, newObj *sourcev1.HelmRepository, chartRepo repository.ChartRepository, res sreconcile.Result, resErr error) { func (r *HelmRepositoryReconciler) notify(ctx context.Context, oldObj, newObj *sourcev1.HelmRepository, chartRepo *repository.ChartRepository, res sreconcile.Result, resErr error) {
// Notify successful reconciliation for new artifact and recovery from any // Notify successful reconciliation for new artifact and recovery from any
// failure. // failure.
if resErr == nil && res == sreconcile.ResultSuccess && newObj.Status.Artifact != nil { if resErr == nil && res == sreconcile.ResultSuccess && newObj.Status.Artifact != nil {
@ -433,7 +435,7 @@ func (r *HelmRepositoryReconciler) reconcileSource(ctx context.Context, sp *patc
} }
// Construct Helm chart repository with options and download index // Construct Helm chart repository with options and download index
newChartRepo, err := repository.NewChartRepository(obj.Spec.URL, "", r.Getters, tlsConfig, clientOpts) newChartRepo, err := repository.NewChartRepository(obj.Spec.URL, "", r.Getters, tlsConfig, clientOpts...)
if err != nil { if err != nil {
switch err.(type) { switch err.(type) {
case *url.Error: case *url.Error:
@ -454,8 +456,7 @@ func (r *HelmRepositoryReconciler) reconcileSource(ctx context.Context, sp *patc
} }
// Fetch the repository index from remote. // Fetch the repository index from remote.
checksum, err := newChartRepo.CacheIndex() if err := newChartRepo.CacheIndex(); err != nil {
if err != nil {
e := &serror.Event{ e := &serror.Event{
Err: fmt.Errorf("failed to fetch Helm repository index: %w", err), Err: fmt.Errorf("failed to fetch Helm repository index: %w", err),
Reason: meta.FailedReason, Reason: meta.FailedReason,
@ -466,20 +467,48 @@ func (r *HelmRepositoryReconciler) reconcileSource(ctx context.Context, sp *patc
} }
*chartRepo = *newChartRepo *chartRepo = *newChartRepo
// Short-circuit based on the fetched index being an exact match to the // Early comparison to current Artifact.
// stored Artifact. This prevents having to unmarshal the YAML to calculate if curArtifact := obj.GetArtifact(); curArtifact != nil {
// the (stable) revision, which is a memory expensive operation. curDig := digestlib.Digest(curArtifact.Digest)
if obj.GetArtifact().HasChecksum(checksum) { if curDig == "" {
*artifact = *obj.GetArtifact() curDig = digestlib.Digest(sourcev1.TransformLegacyRevision(curArtifact.Checksum))
conditions.Delete(obj, sourcev1.FetchFailedCondition) }
return sreconcile.ResultSuccess, nil if curDig.Validate() == nil {
// Short-circuit based on the fetched index being an exact match to the
// stored Artifact. This prevents having to unmarshal the YAML to calculate
// the (stable) revision, which is a memory expensive operation.
if newDig := chartRepo.Digest(curDig.Algorithm()); newDig.Validate() == nil && (newDig == curDig) {
*artifact = *curArtifact
conditions.Delete(obj, sourcev1.FetchFailedCondition)
return sreconcile.ResultSuccess, nil
}
}
} }
// Load the cached repository index to ensure it passes validation. This // Load the cached repository index to ensure it passes validation.
// also populates chartRepo.Checksum. if err := chartRepo.LoadFromPath(); err != nil {
if err := chartRepo.LoadFromCache(); err != nil {
e := &serror.Event{ e := &serror.Event{
Err: fmt.Errorf("failed to load Helm repository from cache: %w", err), Err: fmt.Errorf("failed to load Helm repository from index YAML: %w", err),
Reason: sourcev1.IndexationFailedReason,
}
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
// Delete any stale failure observation
conditions.Delete(obj, sourcev1.FetchFailedCondition)
// Check if index has changed compared to current Artifact revision.
var changed bool
if artifact := obj.Status.Artifact; artifact != nil {
curRev := digestlib.Digest(sourcev1.TransformLegacyRevision(artifact.Revision))
changed = curRev.Validate() != nil || curRev != chartRepo.Revision(curRev.Algorithm())
}
// Calculate revision.
revision := chartRepo.Revision(digest.Canonical)
if revision.Validate() != nil {
e := &serror.Event{
Err: fmt.Errorf("failed to calculate revision: %w", err),
Reason: sourcev1.IndexationFailedReason, Reason: sourcev1.IndexationFailedReason,
} }
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
@ -487,8 +516,8 @@ func (r *HelmRepositoryReconciler) reconcileSource(ctx context.Context, sp *patc
} }
// Mark observations about the revision on the object. // Mark observations about the revision on the object.
if !obj.GetArtifact().HasRevision(chartRepo.Checksum) { if obj.Status.Artifact == nil || changed {
message := fmt.Sprintf("new index revision '%s'", checksum) message := fmt.Sprintf("new index revision '%s'", revision)
if obj.GetArtifact() != nil { if obj.GetArtifact() != nil {
conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", message) conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", message)
} }
@ -500,15 +529,11 @@ func (r *HelmRepositoryReconciler) reconcileSource(ctx context.Context, sp *patc
} }
// Create potential new artifact. // Create potential new artifact.
// Note: Since this is a potential artifact, artifact.Checksum is empty at
// this stage. It's populated when the artifact is written in storage.
*artifact = r.Storage.NewArtifactFor(obj.Kind, *artifact = r.Storage.NewArtifactFor(obj.Kind,
obj.ObjectMeta.GetObjectMeta(), obj.ObjectMeta.GetObjectMeta(),
chartRepo.Checksum, revision.String(),
fmt.Sprintf("index-%s.yaml", checksum)) fmt.Sprintf("index-%s.yaml", revision.Hex()),
)
// Delete any stale failure observation
conditions.Delete(obj, sourcev1.FetchFailedCondition)
return sreconcile.ResultSuccess, nil return sreconcile.ResultSuccess, nil
} }
@ -530,15 +555,17 @@ func (r *HelmRepositoryReconciler) reconcileArtifact(ctx context.Context, sp *pa
conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason, conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason,
"stored artifact: revision '%s'", artifact.Revision) "stored artifact: revision '%s'", artifact.Revision)
} }
if err := chartRepo.Clear(); err != nil {
chartRepo.Unload()
if err := chartRepo.RemoveCache(); err != nil {
ctrl.LoggerFrom(ctx).Error(err, "failed to remove temporary cached index file") ctrl.LoggerFrom(ctx).Error(err, "failed to remove temporary cached index file")
} }
}() }()
if obj.GetArtifact().HasRevision(artifact.Revision) && obj.GetArtifact().HasChecksum(artifact.Checksum) { if obj.GetArtifact().HasRevision(artifact.Revision) && obj.GetArtifact().HasChecksum(artifact.Checksum) {
// Extend TTL of the Index in the cache (if present).
if r.Cache != nil {
r.Cache.SetExpiration(r.Storage.LocalPath(*artifact), r.TTL)
}
r.eventLogf(ctx, obj, eventv1.EventTypeTrace, sourcev1.ArtifactUpToDateReason, "artifact up-to-date with remote revision: '%s'", artifact.Revision) r.eventLogf(ctx, obj, eventv1.EventTypeTrace, sourcev1.ArtifactUpToDateReason, "artifact up-to-date with remote revision: '%s'", artifact.Revision)
return sreconcile.ResultSuccess, nil return sreconcile.ResultSuccess, nil
} }
@ -564,7 +591,7 @@ func (r *HelmRepositoryReconciler) reconcileArtifact(ctx context.Context, sp *pa
defer unlock() defer unlock()
// Save artifact to storage. // Save artifact to storage.
if err = r.Storage.CopyFromPath(artifact, chartRepo.CachePath); err != nil { if err = r.Storage.CopyFromPath(artifact, chartRepo.Path); err != nil {
e := &serror.Event{ e := &serror.Event{
Err: fmt.Errorf("unable to save artifact to storage: %w", err), Err: fmt.Errorf("unable to save artifact to storage: %w", err),
Reason: sourcev1.ArchiveOperationFailedReason, Reason: sourcev1.ArchiveOperationFailedReason,
@ -576,6 +603,18 @@ func (r *HelmRepositoryReconciler) reconcileArtifact(ctx context.Context, sp *pa
// Record it on the object. // Record it on the object.
obj.Status.Artifact = artifact.DeepCopy() obj.Status.Artifact = artifact.DeepCopy()
// Cache the index if it was successfully retrieved.
if r.Cache != nil && chartRepo.Index != nil {
// The cache keys have to be safe in multi-tenancy environments, as
// otherwise it could be used as a vector to bypass the repository's
// authentication. Using r.Storage.LocalPath(*repo.GetArtifact())
// is safe as the path is in the format of:
// /<repository-name>/<chart-name>/<filename>.
if err := r.Cache.Set(r.Storage.LocalPath(*artifact), chartRepo.Index, r.TTL); err != nil {
r.eventLogf(ctx, obj, eventv1.EventTypeTrace, sourcev1.CacheOperationFailedReason, "failed to cache index: %s", err)
}
}
// Update index symlink. // Update index symlink.
indexURL, err := r.Storage.Symlink(*artifact, "index.yaml") indexURL, err := r.Storage.Symlink(*artifact, "index.yaml")
if err != nil { if err != nil {
@ -586,26 +625,6 @@ func (r *HelmRepositoryReconciler) reconcileArtifact(ctx context.Context, sp *pa
obj.Status.URL = indexURL obj.Status.URL = indexURL
} }
conditions.Delete(obj, sourcev1.StorageOperationFailedCondition) conditions.Delete(obj, sourcev1.StorageOperationFailedCondition)
// enable cache if applicable
if r.Cache != nil && chartRepo.IndexCache == nil {
chartRepo.SetMemCache(r.Storage.LocalPath(*artifact), r.Cache, r.TTL, func(event string) {
r.IncCacheEvents(event, obj.GetName(), obj.GetNamespace())
})
}
// Cache the index if it was successfully retrieved
// and the chart was successfully built
if r.Cache != nil && chartRepo.Index != nil {
// The cache key have to be safe in multi-tenancy environments,
// as otherwise it could be used as a vector to bypass the helm repository's authentication.
// Using r.Storage.LocalPath(*repo.GetArtifact() is safe as the path is in the format /<helm-repository-name>/<chart-name>/<filename>.
err := chartRepo.CacheIndexInMemory()
if err != nil {
r.eventLogf(ctx, obj, eventv1.EventTypeTrace, sourcev1.CacheOperationFailedReason, "failed to cache index: %s", err)
}
}
return sreconcile.ResultSuccess, nil return sreconcile.ResultSuccess, nil
} }

226
controllers/helmrepository_controller_test.go
View File

@ -21,6 +21,10 @@ import (
"crypto/tls" "crypto/tls"
"errors" "errors"
"fmt" "fmt"
"github.com/fluxcd/source-controller/internal/cache"
"github.com/fluxcd/source-controller/internal/digest"
digestlib "github.com/opencontainers/go-digest"
"helm.sh/helm/v3/pkg/repo"
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
@ -312,8 +316,8 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
server options server options
url string url string
secret *corev1.Secret secret *corev1.Secret
beforeFunc func(t *WithT, obj *sourcev1.HelmRepository, checksum string) beforeFunc func(t *WithT, obj *sourcev1.HelmRepository, revision, digest digestlib.Digest)
afterFunc func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) afterFunc func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository)
want sreconcile.Result want sreconcile.Result
wantErr bool wantErr bool
assertConditions []metav1.Condition assertConditions []metav1.Condition
@ -344,9 +348,9 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "building artifact: new index revision"), *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "building artifact: new index revision"),
*conditions.UnknownCondition(meta.ReadyCondition, meta.ProgressingReason, "building artifact: new index revision"), *conditions.UnknownCondition(meta.ReadyCondition, meta.ProgressingReason, "building artifact: new index revision"),
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
t.Expect(chartRepo.Checksum).ToNot(BeEmpty()) t.Expect(chartRepo.Path).ToNot(BeEmpty())
t.Expect(chartRepo.CachePath).ToNot(BeEmpty()) t.Expect(chartRepo.Index).ToNot(BeNil())
t.Expect(artifact.Checksum).To(BeEmpty()) t.Expect(artifact.Checksum).To(BeEmpty())
t.Expect(artifact.Revision).ToNot(BeEmpty()) t.Expect(artifact.Revision).ToNot(BeEmpty())
}, },
@ -367,7 +371,7 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
"password": []byte("1234"), "password": []byte("1234"),
}, },
}, },
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, checksum string) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, checksum digestlib.Digest) {
obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "basic-auth"} obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "basic-auth"}
}, },
want: sreconcile.ResultSuccess, want: sreconcile.ResultSuccess,
@ -375,9 +379,9 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "building artifact: new index revision"), *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "building artifact: new index revision"),
*conditions.UnknownCondition(meta.ReadyCondition, meta.ProgressingReason, "building artifact: new index revision"), *conditions.UnknownCondition(meta.ReadyCondition, meta.ProgressingReason, "building artifact: new index revision"),
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
t.Expect(chartRepo.Checksum).ToNot(BeEmpty()) t.Expect(chartRepo.Path).ToNot(BeEmpty())
t.Expect(chartRepo.CachePath).ToNot(BeEmpty()) t.Expect(chartRepo.Index).ToNot(BeNil())
t.Expect(artifact.Checksum).To(BeEmpty()) t.Expect(artifact.Checksum).To(BeEmpty())
t.Expect(artifact.Revision).ToNot(BeEmpty()) t.Expect(artifact.Revision).ToNot(BeEmpty())
}, },
@ -398,7 +402,7 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
"caFile": tlsCA, "caFile": tlsCA,
}, },
}, },
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, checksum string) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, checksum digestlib.Digest) {
obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "ca-file"} obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "ca-file"}
}, },
want: sreconcile.ResultSuccess, want: sreconcile.ResultSuccess,
@ -406,9 +410,9 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "building artifact: new index revision"), *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "building artifact: new index revision"),
*conditions.UnknownCondition(meta.ReadyCondition, meta.ProgressingReason, "building artifact: new index revision"), *conditions.UnknownCondition(meta.ReadyCondition, meta.ProgressingReason, "building artifact: new index revision"),
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
t.Expect(chartRepo.Checksum).ToNot(BeEmpty()) t.Expect(chartRepo.Path).ToNot(BeEmpty())
t.Expect(chartRepo.CachePath).ToNot(BeEmpty()) t.Expect(chartRepo.Index).ToNot(BeNil())
t.Expect(artifact.Checksum).To(BeEmpty()) t.Expect(artifact.Checksum).To(BeEmpty())
t.Expect(artifact.Revision).ToNot(BeEmpty()) t.Expect(artifact.Revision).ToNot(BeEmpty())
}, },
@ -429,7 +433,7 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
"caFile": []byte("invalid"), "caFile": []byte("invalid"),
}, },
}, },
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, checksum string) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, checksum digestlib.Digest) {
obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "invalid-ca"} obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "invalid-ca"}
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo") conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar") conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
@ -440,10 +444,10 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"), *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"), *conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
// No repo index due to fetch fail. // No repo index due to fetch fail.
t.Expect(chartRepo.Checksum).To(BeEmpty()) t.Expect(chartRepo.Path).To(BeEmpty())
t.Expect(chartRepo.CachePath).To(BeEmpty()) t.Expect(chartRepo.Index).To(BeNil())
t.Expect(artifact.Checksum).To(BeEmpty()) t.Expect(artifact.Checksum).To(BeEmpty())
t.Expect(artifact.Revision).To(BeEmpty()) t.Expect(artifact.Revision).To(BeEmpty())
}, },
@ -451,7 +455,7 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
{ {
name: "Invalid URL makes FetchFailed=True and returns stalling error", name: "Invalid URL makes FetchFailed=True and returns stalling error",
protocol: "http", protocol: "http",
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, checksum string) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, checksum digestlib.Digest) {
obj.Spec.URL = strings.ReplaceAll(obj.Spec.URL, "http://", "") obj.Spec.URL = strings.ReplaceAll(obj.Spec.URL, "http://", "")
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo") conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar") conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
@ -463,10 +467,10 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"), *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"), *conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
// No repo index due to fetch fail. // No repo index due to fetch fail.
t.Expect(chartRepo.Checksum).To(BeEmpty()) t.Expect(chartRepo.Path).To(BeEmpty())
t.Expect(chartRepo.CachePath).To(BeEmpty()) t.Expect(chartRepo.Index).To(BeNil())
t.Expect(artifact.Checksum).To(BeEmpty()) t.Expect(artifact.Checksum).To(BeEmpty())
t.Expect(artifact.Revision).To(BeEmpty()) t.Expect(artifact.Revision).To(BeEmpty())
}, },
@ -474,7 +478,7 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
{ {
name: "Unsupported scheme makes FetchFailed=True and returns stalling error", name: "Unsupported scheme makes FetchFailed=True and returns stalling error",
protocol: "http", protocol: "http",
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, checksum string) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, checksum digestlib.Digest) {
obj.Spec.URL = strings.ReplaceAll(obj.Spec.URL, "http://", "ftp://") obj.Spec.URL = strings.ReplaceAll(obj.Spec.URL, "http://", "ftp://")
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo") conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar") conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
@ -486,10 +490,10 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"), *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"), *conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
// No repo index due to fetch fail. // No repo index due to fetch fail.
t.Expect(chartRepo.Checksum).To(BeEmpty()) t.Expect(chartRepo.Path).To(BeEmpty())
t.Expect(chartRepo.CachePath).To(BeEmpty()) t.Expect(chartRepo.Index).To(BeNil())
t.Expect(artifact.Checksum).To(BeEmpty()) t.Expect(artifact.Checksum).To(BeEmpty())
t.Expect(artifact.Revision).To(BeEmpty()) t.Expect(artifact.Revision).To(BeEmpty())
}, },
@ -497,7 +501,7 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
{ {
name: "Missing secret returns FetchFailed=True and returns error", name: "Missing secret returns FetchFailed=True and returns error",
protocol: "http", protocol: "http",
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, checksum string) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, checksum digestlib.Digest) {
obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "non-existing"} obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "non-existing"}
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo") conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar") conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
@ -508,10 +512,10 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"), *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"), *conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
// No repo index due to fetch fail. // No repo index due to fetch fail.
t.Expect(chartRepo.Checksum).To(BeEmpty()) t.Expect(chartRepo.Path).To(BeEmpty())
t.Expect(chartRepo.CachePath).To(BeEmpty()) t.Expect(chartRepo.Index).To(BeNil())
t.Expect(artifact.Checksum).To(BeEmpty()) t.Expect(artifact.Checksum).To(BeEmpty())
t.Expect(artifact.Revision).To(BeEmpty()) t.Expect(artifact.Revision).To(BeEmpty())
}, },
@ -527,7 +531,7 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
"username": []byte("git"), "username": []byte("git"),
}, },
}, },
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, checksum string) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, checksum digestlib.Digest) {
obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "malformed-basic-auth"} obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "malformed-basic-auth"}
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo") conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar") conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
@ -538,66 +542,125 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"), *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"), *conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
// No repo index due to fetch fail. // No repo index due to fetch fail.
t.Expect(chartRepo.Checksum).To(BeEmpty()) t.Expect(chartRepo.Path).To(BeEmpty())
t.Expect(chartRepo.CachePath).To(BeEmpty()) t.Expect(chartRepo.Index).To(BeNil())
t.Expect(artifact.Checksum).To(BeEmpty()) t.Expect(artifact.Checksum).To(BeEmpty())
t.Expect(artifact.Revision).To(BeEmpty()) t.Expect(artifact.Revision).To(BeEmpty())
}, },
}, },
{ {
name: "cached index with same checksum", name: "Stored index with same digest and revision",
protocol: "http", protocol: "http",
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, checksum string) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, digest digestlib.Digest) {
obj.Status.Artifact = &sourcev1.Artifact{ obj.Status.Artifact = &sourcev1.Artifact{
Revision: checksum, Revision: revision.String(),
Checksum: checksum, Digest: digest.String(),
Checksum: digest.Hex(),
} }
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo") conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar") conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, "foo", "bar")
}, },
assertConditions: []metav1.Condition{ assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"), *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"), *conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
// chartRepo.Checksum isn't populated, artifact.Checksum is t.Expect(chartRepo.Path).ToNot(BeEmpty())
// populated from the cached repo index data. t.Expect(chartRepo.Index).To(BeNil())
t.Expect(chartRepo.Checksum).To(BeEmpty())
t.Expect(chartRepo.CachePath).ToNot(BeEmpty()) t.Expect(&artifact).To(BeEquivalentTo(obj.Status.Artifact))
t.Expect(artifact.Checksum).To(Equal(obj.Status.Artifact.Checksum))
t.Expect(artifact.Revision).To(Equal(obj.Status.Artifact.Revision))
}, },
want: sreconcile.ResultSuccess, want: sreconcile.ResultSuccess,
}, },
{ {
name: "cached index with different checksum", name: "Stored index with same checksum and (legacy) revision",
protocol: "http", protocol: "http",
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, checksum string) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, digest digestlib.Digest) {
obj.Status.Artifact = &sourcev1.Artifact{ obj.Status.Artifact = &sourcev1.Artifact{
Revision: checksum, Revision: revision.Hex(),
Checksum: "foo", Checksum: digest.Hex(),
} }
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo") conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar") conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, "foo", "bar")
}, },
assertConditions: []metav1.Condition{ assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"), *conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"), *conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo repository.ChartRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
t.Expect(chartRepo.Checksum).ToNot(BeEmpty()) t.Expect(chartRepo.Path).ToNot(BeEmpty())
t.Expect(chartRepo.CachePath).ToNot(BeEmpty()) t.Expect(chartRepo.Index).To(BeNil())
t.Expect(artifact.Checksum).To(BeEmpty())
t.Expect(&artifact).To(BeEquivalentTo(obj.Status.Artifact))
},
want: sreconcile.ResultSuccess,
},
{
name: "Stored index with different digest and same revision",
protocol: "http",
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, digest digestlib.Digest) {
obj.Status.Artifact = &sourcev1.Artifact{
Revision: revision.String(),
Digest: "sha256:80bb3dd67c63095d985850459834ea727603727a370079de90d221191d375a86",
Checksum: "80bb3dd67c63095d985850459834ea727603727a370079de90d221191d375a86",
}
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, "foo", "bar")
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "foo"),
*conditions.UnknownCondition(meta.ReadyCondition, "foo", "bar"),
},
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
t.Expect(chartRepo.Path).ToNot(BeEmpty())
t.Expect(chartRepo.Index).ToNot(BeNil())
t.Expect(artifact.Revision).To(Equal(obj.Status.Artifact.Revision)) t.Expect(artifact.Revision).To(Equal(obj.Status.Artifact.Revision))
t.Expect(artifact.Digest).ToNot(Equal(obj.Status.Artifact.Digest))
t.Expect(artifact.Checksum).ToNot(Equal(obj.Status.Artifact.Checksum))
},
want: sreconcile.ResultSuccess,
},
{
name: "Stored index with different revision and digest",
protocol: "http",
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, checksum digestlib.Digest) {
obj.Status.Artifact = &sourcev1.Artifact{
Revision: "80bb3dd67c63095d985850459834ea727603727a370079de90d221191d375a86",
Checksum: "80bb3dd67c63095d985850459834ea727603727a370079de90d221191d375a86",
Digest: "sha256:80bb3dd67c63095d985850459834ea727603727a370079de90d221191d375a86",
}
conditions.MarkReconciling(obj, meta.ProgressingReason, "foo")
conditions.MarkUnknown(obj, meta.ReadyCondition, "foo", "bar")
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new index revision"),
*conditions.TrueCondition(meta.ReconcilingCondition, meta.ProgressingReason, "building artifact: new index revision"),
*conditions.UnknownCondition(meta.ReadyCondition, meta.ProgressingReason, "building artifact: new index revision"),
},
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, chartRepo *repository.ChartRepository) {
t.Expect(chartRepo.Path).ToNot(BeEmpty())
t.Expect(chartRepo.Index).ToNot(BeNil())
t.Expect(artifact.Path).To(Not(BeEmpty()))
t.Expect(artifact.Revision).ToNot(Equal(obj.Status.Artifact.Revision))
t.Expect(artifact.Digest).ToNot(Equal(obj.Status.Artifact.Digest))
t.Expect(artifact.Checksum).ToNot(Equal(obj.Status.Artifact.Checksum))
}, },
want: sreconcile.ResultSuccess, want: sreconcile.ResultSuccess,
}, },
{ {
name: "Existing artifact makes ArtifactOutdated=True", name: "Existing artifact makes ArtifactOutdated=True",
protocol: "http", protocol: "http",
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, checksum string) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, revision, checksum digestlib.Digest) {
obj.Status.Artifact = &sourcev1.Artifact{ obj.Status.Artifact = &sourcev1.Artifact{
Path: "some-path", Path: "some-path",
Revision: "some-rev", Revision: "some-rev",
@ -698,22 +761,24 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
if serr != nil { if serr != nil {
validSecret = false validSecret = false
} }
newChartRepo, err = repository.NewChartRepository(obj.Spec.URL, "", testGetters, tOpts, clientOpts) newChartRepo, err = repository.NewChartRepository(obj.Spec.URL, "", testGetters, tOpts, clientOpts...)
} else { } else {
newChartRepo, err = repository.NewChartRepository(obj.Spec.URL, "", testGetters, nil, nil) newChartRepo, err = repository.NewChartRepository(obj.Spec.URL, "", testGetters, nil)
} }
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(HaveOccurred())
// NOTE: checksum will be empty in beforeFunc for invalid repo // NOTE: checksum will be empty in beforeFunc for invalid repo
// configurations as the client can't get the repo. // configurations as the client can't get the repo.
var indexChecksum string var revision, checksum digestlib.Digest
if validSecret { if validSecret {
indexChecksum, err = newChartRepo.CacheIndex() g.Expect(newChartRepo.CacheIndex()).To(Succeed())
g.Expect(err).ToNot(HaveOccurred()) checksum = newChartRepo.Digest(digest.Canonical)
}
g.Expect(newChartRepo.LoadFromPath()).To(Succeed())
revision = newChartRepo.Revision(digest.Canonical)
}
if tt.beforeFunc != nil { if tt.beforeFunc != nil {
tt.beforeFunc(g, obj, indexChecksum) tt.beforeFunc(g, obj, revision, checksum)
} }
r := &HelmRepositoryReconciler{ r := &HelmRepositoryReconciler{
@ -734,14 +799,14 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
sp := patch.NewSerialPatcher(obj, r.Client) sp := patch.NewSerialPatcher(obj, r.Client)
got, err := r.reconcileSource(context.TODO(), sp, obj, &artifact, &chartRepo) got, err := r.reconcileSource(context.TODO(), sp, obj, &artifact, &chartRepo)
defer os.Remove(chartRepo.CachePath) defer os.Remove(chartRepo.Path)
g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions)) g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
g.Expect(err != nil).To(Equal(tt.wantErr)) g.Expect(err != nil).To(Equal(tt.wantErr))
g.Expect(got).To(Equal(tt.want)) g.Expect(got).To(Equal(tt.want))
if tt.afterFunc != nil { if tt.afterFunc != nil {
tt.afterFunc(g, obj, artifact, chartRepo) tt.afterFunc(g, obj, artifact, &chartRepo)
} }
// In-progress status condition validity. // In-progress status condition validity.
@ -754,8 +819,9 @@ func TestHelmRepositoryReconciler_reconcileSource(t *testing.T) {
func TestHelmRepositoryReconciler_reconcileArtifact(t *testing.T) { func TestHelmRepositoryReconciler_reconcileArtifact(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
cache *cache.Cache
beforeFunc func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, index *repository.ChartRepository) beforeFunc func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, index *repository.ChartRepository)
afterFunc func(t *WithT, obj *sourcev1.HelmRepository) afterFunc func(t *WithT, obj *sourcev1.HelmRepository, cache *cache.Cache)
want sreconcile.Result want sreconcile.Result
wantErr bool wantErr bool
assertConditions []metav1.Condition assertConditions []metav1.Condition
@ -770,13 +836,33 @@ func TestHelmRepositoryReconciler_reconcileArtifact(t *testing.T) {
*conditions.TrueCondition(sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "stored artifact: revision 'existing'"), *conditions.TrueCondition(sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "stored artifact: revision 'existing'"),
}, },
}, },
{
name: "Archiving (loaded) artifact to storage adds to cache",
cache: cache.New(10, time.Minute),
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, index *repository.ChartRepository) {
index.Index = &repo.IndexFile{
APIVersion: "v1",
Generated: time.Now(),
}
obj.Spec.Interval = metav1.Duration{Duration: interval}
},
want: sreconcile.ResultSuccess,
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, cache *cache.Cache) {
i, ok := cache.Get(testStorage.LocalPath(*obj.GetArtifact()))
t.Expect(ok).To(BeTrue())
t.Expect(i).To(BeAssignableToTypeOf(&repo.IndexFile{}))
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "stored artifact: revision 'existing'"),
},
},
{ {
name: "Up-to-date artifact should not update status", name: "Up-to-date artifact should not update status",
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, index *repository.ChartRepository) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, index *repository.ChartRepository) {
obj.Spec.Interval = metav1.Duration{Duration: interval} obj.Spec.Interval = metav1.Duration{Duration: interval}
obj.Status.Artifact = artifact.DeepCopy() obj.Status.Artifact = artifact.DeepCopy()
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, _ *cache.Cache) {
t.Expect(obj.Status.URL).To(BeEmpty()) t.Expect(obj.Status.URL).To(BeEmpty())
}, },
want: sreconcile.ResultSuccess, want: sreconcile.ResultSuccess,
@ -800,7 +886,7 @@ func TestHelmRepositoryReconciler_reconcileArtifact(t *testing.T) {
beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, index *repository.ChartRepository) { beforeFunc: func(t *WithT, obj *sourcev1.HelmRepository, artifact sourcev1.Artifact, index *repository.ChartRepository) {
obj.Spec.Interval = metav1.Duration{Duration: interval} obj.Spec.Interval = metav1.Duration{Duration: interval}
}, },
afterFunc: func(t *WithT, obj *sourcev1.HelmRepository) { afterFunc: func(t *WithT, obj *sourcev1.HelmRepository, _ *cache.Cache) {
localPath := testStorage.LocalPath(*obj.GetArtifact()) localPath := testStorage.LocalPath(*obj.GetArtifact())
symlinkPath := filepath.Join(filepath.Dir(localPath), "index.yaml") symlinkPath := filepath.Join(filepath.Dir(localPath), "index.yaml")
targetFile, err := os.Readlink(symlinkPath) targetFile, err := os.Readlink(symlinkPath)
@ -822,6 +908,8 @@ func TestHelmRepositoryReconciler_reconcileArtifact(t *testing.T) {
Client: fakeclient.NewClientBuilder().WithScheme(testEnv.GetScheme()).Build(), Client: fakeclient.NewClientBuilder().WithScheme(testEnv.GetScheme()).Build(),
EventRecorder: record.NewFakeRecorder(32), EventRecorder: record.NewFakeRecorder(32),
Storage: testStorage, Storage: testStorage,
Cache: tt.cache,
TTL: 1 * time.Minute,
patchOptions: getPatchOptions(helmRepositoryReadyCondition.Owned, "sc"), patchOptions: getPatchOptions(helmRepositoryReadyCondition.Owned, "sc"),
} }
@ -848,9 +936,9 @@ func TestHelmRepositoryReconciler_reconcileArtifact(t *testing.T) {
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(HaveOccurred())
g.Expect(cacheFile.Close()).ToNot(HaveOccurred()) g.Expect(cacheFile.Close()).ToNot(HaveOccurred())
chartRepo, err := repository.NewChartRepository(obj.Spec.URL, "", testGetters, nil, nil) chartRepo, err := repository.NewChartRepository(obj.Spec.URL, "", testGetters, nil)
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(HaveOccurred())
chartRepo.CachePath = cachePath chartRepo.Path = cachePath
artifact := testStorage.NewArtifactFor(obj.Kind, obj, "existing", "foo.tar.gz") artifact := testStorage.NewArtifactFor(obj.Kind, obj, "existing", "foo.tar.gz")
// Checksum of the index file calculated by the ChartRepository. // Checksum of the index file calculated by the ChartRepository.
@ -873,7 +961,7 @@ func TestHelmRepositoryReconciler_reconcileArtifact(t *testing.T) {
g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions)) g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions))
if tt.afterFunc != nil { if tt.afterFunc != nil {
tt.afterFunc(g, obj) tt.afterFunc(g, obj, tt.cache)
} }
}) })
} }
@ -1209,7 +1297,7 @@ func TestHelmRepositoryReconciler_notify(t *testing.T) {
chartRepo := repository.ChartRepository{ chartRepo := repository.ChartRepository{
URL: "some-address", URL: "some-address",
} }
reconciler.notify(ctx, oldObj, newObj, chartRepo, tt.res, tt.resErr) reconciler.notify(ctx, oldObj, newObj, &chartRepo, tt.res, tt.resErr)
select { select {
case x, ok := <-recorder.Events: case x, ok := <-recorder.Events:

9
internal/helm/chart/builder_remote_test.go
View File

@ -193,10 +193,9 @@ entries:
targetPath := filepath.Join(tmpDir, "chart.tgz") targetPath := filepath.Join(tmpDir, "chart.tgz")
if tt.repository != nil { if tt.repository != nil {
_, err := tt.repository.CacheIndex() g.Expect(tt.repository.CacheIndex()).ToNot(HaveOccurred())
g.Expect(err).ToNot(HaveOccurred())
// Cleanup the cache index path. // Cleanup the cache index path.
defer os.Remove(tt.repository.CachePath) defer os.Remove(tt.repository.Path)
} }
b := NewRemoteBuilder(tt.repository) b := NewRemoteBuilder(tt.repository)
@ -411,10 +410,10 @@ entries:
reference := RemoteReference{Name: "helmchart"} reference := RemoteReference{Name: "helmchart"}
repository := mockRepo() repository := mockRepo()
_, err = repository.CacheIndex() err = repository.CacheIndex()
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(HaveOccurred())
// Cleanup the cache index path. // Cleanup the cache index path.
defer os.Remove(repository.CachePath) defer os.Remove(repository.Path)
b := NewRemoteBuilder(repository) b := NewRemoteBuilder(repository)

13
internal/helm/chart/dependency_manager_test.go
View File

@ -86,11 +86,6 @@ func TestDependencyManager_Clear(t *testing.T) {
Index: repo.NewIndexFile(), Index: repo.NewIndexFile(),
RWMutex: &sync.RWMutex{}, RWMutex: &sync.RWMutex{},
}, },
"cached cache path": &repository.ChartRepository{
CachePath: "/invalid/path/resets",
Cached: true,
RWMutex: &sync.RWMutex{},
},
"with credentials": ociRepoWithCreds, "with credentials": ociRepoWithCreds,
"without credentials": &repository.OCIChartRepository{}, "without credentials": &repository.OCIChartRepository{},
"nil downloader": nil, "nil downloader": nil,
@ -103,8 +98,6 @@ func TestDependencyManager_Clear(t *testing.T) {
switch v := v.(type) { switch v := v.(type) {
case *repository.ChartRepository: case *repository.ChartRepository:
g.Expect(v.Index).To(BeNil()) g.Expect(v.Index).To(BeNil())
g.Expect(v.CachePath).To(BeEmpty())
g.Expect(v.Cached).To(BeFalse())
case *repository.OCIChartRepository: case *repository.OCIChartRepository:
g.Expect(v.HasCredentials()).To(BeFalse()) g.Expect(v.HasCredentials()).To(BeFalse())
} }
@ -441,14 +434,14 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) {
name: "strategic load error", name: "strategic load error",
downloaders: map[string]repository.Downloader{ downloaders: map[string]repository.Downloader{
"https://example.com/": &repository.ChartRepository{ "https://example.com/": &repository.ChartRepository{
CachePath: "/invalid/cache/path/foo", Client: &mockGetter{},
RWMutex: &sync.RWMutex{}, RWMutex: &sync.RWMutex{},
}, },
}, },
dep: &helmchart.Dependency{ dep: &helmchart.Dependency{
Repository: "https://example.com", Repository: "https://example.com",
}, },
wantErr: "failed to strategically load index", wantErr: "failed to load index",
}, },
{ {
name: "repository get error", name: "repository get error",

427
internal/helm/repository/chart_repository.go
View File

@ -19,12 +19,9 @@ package repository
import ( import (
"bytes" "bytes"
"context" "context"
"crypto/sha256"
"crypto/tls" "crypto/tls"
"encoding/hex"
"errors" "errors"
"fmt" "fmt"
"github.com/opencontainers/go-digest"
"io" "io"
"net/url" "net/url"
"os" "os"
@ -32,22 +29,79 @@ import (
"sort" "sort"
"strings" "strings"
"sync" "sync"
"time"
"github.com/Masterminds/semver/v3" "github.com/Masterminds/semver/v3"
"github.com/opencontainers/go-digest"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/getter"
"helm.sh/helm/v3/pkg/repo" "helm.sh/helm/v3/pkg/repo"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"sigs.k8s.io/yaml" "sigs.k8s.io/yaml"
"github.com/fluxcd/pkg/version" "github.com/fluxcd/pkg/version"
"github.com/fluxcd/source-controller/internal/cache"
"github.com/fluxcd/source-controller/internal/helm" "github.com/fluxcd/source-controller/internal/helm"
"github.com/fluxcd/source-controller/internal/transport" "github.com/fluxcd/source-controller/internal/transport"
) )
var ErrNoChartIndex = errors.New("no chart index") var (
ErrNoChartIndex = errors.New("no chart index")
)
// IndexFromFile loads a repo.IndexFile from the given path. It returns an
// error if the file does not exist, is not a regular file, exceeds the
// maximum index file size, or if the file cannot be parsed.
func IndexFromFile(path string) (*repo.IndexFile, error) {
st, err := os.Lstat(path)
if err != nil {
return nil, err
}
if !st.Mode().IsRegular() {
return nil, fmt.Errorf("%s is not a regular file", path)
}
if st.Size() > helm.MaxIndexSize {
return nil, fmt.Errorf("%s exceeds the maximum index file size of %d bytes", path, helm.MaxIndexSize)
}
b, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return IndexFromBytes(b)
}
// IndexFromBytes loads a repo.IndexFile from the given bytes. It returns an
// error if the bytes cannot be parsed, or if the API version is not set.
// The entries are sorted before the index is returned.
func IndexFromBytes(b []byte) (*repo.IndexFile, error) {
if len(b) == 0 {
return nil, repo.ErrEmptyIndexYaml
}
i := &repo.IndexFile{}
if err := yaml.UnmarshalStrict(b, i); err != nil {
return nil, err
}
if i.APIVersion == "" {
return nil, repo.ErrNoAPIVersion
}
for _, cvs := range i.Entries {
for idx := len(cvs) - 1; idx >= 0; idx-- {
if cvs[idx] == nil {
continue
}
if cvs[idx].APIVersion == "" {
cvs[idx].APIVersion = chart.APIVersionV1
}
if err := cvs[idx].Validate(); err != nil {
cvs = append(cvs[:idx], cvs[idx+1:]...)
}
}
}
i.SortEntries()
return i, nil
}
// ChartRepository represents a Helm chart repository, and the configuration // ChartRepository represents a Helm chart repository, and the configuration
// required to download the chart index and charts from the repository. // required to download the chart index and charts from the repository.
@ -56,73 +110,32 @@ type ChartRepository struct {
// URL the ChartRepository's index.yaml can be found at, // URL the ChartRepository's index.yaml can be found at,
// without the index.yaml suffix. // without the index.yaml suffix.
URL string URL string
// Path is the absolute path to the Index file.
Path string
// Index of the ChartRepository.
Index *repo.IndexFile
// Client to use while downloading the Index or a chart from the URL. // Client to use while downloading the Index or a chart from the URL.
Client getter.Getter Client getter.Getter
// Options to configure the Client with while downloading the Index // Options to configure the Client with while downloading the Index
// or a chart from the URL. // or a chart from the URL.
Options []getter.Option Options []getter.Option
// CachePath is the path of a cached index.yaml for read-only operations.
CachePath string
// Cached indicates if the ChartRepository index.yaml has been cached
// to CachePath.
Cached bool
// Index contains a loaded chart repository index if not nil.
Index *repo.IndexFile
// Checksum contains the SHA256 checksum of the loaded chart repository
// index bytes. This is different from the checksum of the CachePath, which
// may contain unordered entries.
Checksum string
tlsConfig *tls.Config tlsConfig *tls.Config
cached bool
revisions map[digest.Algorithm]digest.Digest
digests map[digest.Algorithm]digest.Digest
*sync.RWMutex *sync.RWMutex
cacheInfo
}
type cacheInfo struct {
// In memory cache of the index.yaml file.
IndexCache *cache.Cache
// IndexKey is the cache key for the index.yaml file.
IndexKey string
// IndexTTL is the cache TTL for the index.yaml file.
IndexTTL time.Duration
// RecordIndexCacheMetric records the cache hit/miss metrics for the index.yaml file.
RecordIndexCacheMetric RecordMetricsFunc
}
// ChartRepositoryOption is a function that can be passed to NewChartRepository
// to configure a ChartRepository.
type ChartRepositoryOption func(*ChartRepository) error
// RecordMetricsFunc is a function that records metrics.
type RecordMetricsFunc func(event string)
// WithMemoryCache returns a ChartRepositoryOptions that will enable the
// ChartRepository to cache the index.yaml file in memory.
// The cache key have to be safe in multi-tenancy environments,
// as otherwise it could be used as a vector to bypass the helm repository's authentication.
func WithMemoryCache(key string, c *cache.Cache, ttl time.Duration, rec RecordMetricsFunc) ChartRepositoryOption {
return func(r *ChartRepository) error {
if c != nil {
if key == "" {
return errors.New("cache key cannot be empty")
}
}
r.IndexCache = c
r.IndexKey = key
r.IndexTTL = ttl
r.RecordIndexCacheMetric = rec
return nil
}
} }
// NewChartRepository constructs and returns a new ChartRepository with // NewChartRepository constructs and returns a new ChartRepository with
// the ChartRepository.Client configured to the getter.Getter for the // the ChartRepository.Client configured to the getter.Getter for the
// repository URL scheme. It returns an error on URL parsing failures, // repository URL scheme. It returns an error on URL parsing failures,
// or if there is no getter available for the scheme. // or if there is no getter available for the scheme.
func NewChartRepository(repositoryURL, cachePath string, providers getter.Providers, tlsConfig *tls.Config, getterOpts []getter.Option, chartRepoOpts ...ChartRepositoryOption) (*ChartRepository, error) { func NewChartRepository(URL, path string, providers getter.Providers, tlsConfig *tls.Config, getterOpts ...getter.Option) (*ChartRepository, error) {
u, err := url.Parse(repositoryURL) u, err := url.Parse(URL)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -132,24 +145,20 @@ func NewChartRepository(repositoryURL, cachePath string, providers getter.Provid
} }
r := newChartRepository() r := newChartRepository()
r.URL = repositoryURL r.URL = URL
r.CachePath = cachePath r.Path = path
r.Client = c r.Client = c
r.Options = getterOpts r.Options = getterOpts
r.tlsConfig = tlsConfig r.tlsConfig = tlsConfig
for _, opt := range chartRepoOpts {
if err := opt(r); err != nil {
return nil, err
}
}
return r, nil return r, nil
} }
func newChartRepository() *ChartRepository { func newChartRepository() *ChartRepository {
return &ChartRepository{ return &ChartRepository{
RWMutex: &sync.RWMutex{}, revisions: make(map[digest.Algorithm]digest.Digest, 0),
digests: make(map[digest.Algorithm]digest.Digest, 0),
RWMutex: &sync.RWMutex{},
} }
} }
@ -206,10 +215,10 @@ func (r *ChartRepository) getChartVersion(name, ver string) (*repo.ChartVersion,
} }
} }
// Filter out chart versions that doesn't satisfy constraints if any, // Filter out chart versions that don't satisfy constraints if any,
// parse semver and build a lookup table // parse semver and build a lookup table
var matchedVersions semver.Collection var matchedVersions semver.Collection
lookup := make(map[*semver.Version]*repo.ChartVersion) lookup := make(map[*semver.Version]*repo.ChartVersion, 0)
for _, cv := range cvs { for _, cv := range cvs {
v, err := version.ParseVersion(cv.Version) v, err := version.ParseVersion(cv.Version)
if err != nil { if err != nil {
@ -289,155 +298,86 @@ func (r *ChartRepository) DownloadChart(chart *repo.ChartVersion) (*bytes.Buffer
return r.Client.Get(u.String(), clientOpts...) return r.Client.Get(u.String(), clientOpts...)
} }
// LoadIndexFromBytes loads Index from the given bytes.
// It returns a repo.ErrNoAPIVersion error if the API version is not set
func (r *ChartRepository) LoadIndexFromBytes(b []byte) error {
i := &repo.IndexFile{}
if err := yaml.UnmarshalStrict(b, i); err != nil {
return err
}
if i.APIVersion == "" {
return repo.ErrNoAPIVersion
}
i.SortEntries()
r.Lock()
r.Index = i
r.Checksum = digest.SHA256.FromBytes(b).Hex()
r.Unlock()
return nil
}
// LoadFromFile reads the file at the given path and loads it into Index.
func (r *ChartRepository) LoadFromFile(path string) error {
stat, err := os.Stat(path)
if err != nil || stat.IsDir() {
if err == nil {
err = fmt.Errorf("'%s' is a directory", path)
}
return err
}
if stat.Size() > helm.MaxIndexSize {
return fmt.Errorf("size of index '%s' exceeds '%d' bytes limit", stat.Name(), helm.MaxIndexSize)
}
b, err := os.ReadFile(path)
if err != nil {
return err
}
return r.LoadIndexFromBytes(b)
}
// CacheIndex attempts to write the index from the remote into a new temporary file // CacheIndex attempts to write the index from the remote into a new temporary file
// using DownloadIndex, and sets CachePath and Cached. // using DownloadIndex, and sets Path and cached.
// It returns the SHA256 checksum of the downloaded index bytes, or an error. // It returns the SHA256 checksum of the downloaded index bytes, or an error.
// The caller is expected to handle the garbage collection of CachePath, and to // The caller is expected to handle the garbage collection of Path, and to
// load the Index separately using LoadFromCache if required. // load the Index separately using LoadFromPath if required.
func (r *ChartRepository) CacheIndex() (string, error) { func (r *ChartRepository) CacheIndex() error {
f, err := os.CreateTemp("", "chart-index-*.yaml") f, err := os.CreateTemp("", "chart-index-*.yaml")
if err != nil { if err != nil {
return "", fmt.Errorf("failed to create temp file to cache index to: %w", err) return fmt.Errorf("failed to create temp file to cache index to: %w", err)
} }
h := sha256.New() if err = r.DownloadIndex(f); err != nil {
mw := io.MultiWriter(f, h)
if err = r.DownloadIndex(mw); err != nil {
f.Close() f.Close()
os.RemoveAll(f.Name()) os.Remove(f.Name())
return "", fmt.Errorf("failed to cache index to temporary file: %w", err) return fmt.Errorf("failed to cache index to temporary file: %w", err)
} }
if err = f.Close(); err != nil { if err = f.Close(); err != nil {
os.RemoveAll(f.Name()) os.Remove(f.Name())
return "", fmt.Errorf("failed to close cached index file '%s': %w", f.Name(), err) return fmt.Errorf("failed to close cached index file '%s': %w", f.Name(), err)
} }
r.Lock() r.Lock()
r.CachePath = f.Name() r.Path = f.Name()
r.Cached = true r.Index = nil
r.cached = true
r.invalidate()
r.Unlock() r.Unlock()
return hex.EncodeToString(h.Sum(nil)), nil
}
// CacheIndexInMemory attempts to cache the index in memory.
// It returns an error if it fails.
// The cache key have to be safe in multi-tenancy environments,
// as otherwise it could be used as a vector to bypass the helm repository's authentication.
func (r *ChartRepository) CacheIndexInMemory() error {
// Cache the index if it was successfully retrieved
// and the chart was successfully built
if r.IndexCache != nil && r.Index != nil {
err := r.IndexCache.Set(r.IndexKey, r.Index, r.IndexTTL)
if err != nil {
return err
}
}
return nil return nil
} }
// StrategicallyLoadIndex lazy-loads the Index // StrategicallyLoadIndex lazy-loads the Index if required, first
// first from Indexcache, // attempting to load it from Path if the file exists, before falling
// then from CachePath using oadFromCache if it does not HasIndex. // back to caching it.
// If not HasCacheFile, a cache attempt is made using CacheIndex
// before continuing to load.
func (r *ChartRepository) StrategicallyLoadIndex() (err error) { func (r *ChartRepository) StrategicallyLoadIndex() (err error) {
if r.HasIndex() { if r.HasIndex() {
return return
} }
if r.IndexCache != nil { if !r.HasFile() {
if found := r.LoadFromMemCache(); found { if err = r.CacheIndex(); err != nil {
err = fmt.Errorf("failed to cache index: %w", err)
return return
} }
} }
if !r.HasCacheFile() { if err = r.LoadFromPath(); err != nil {
if _, err = r.CacheIndex(); err != nil { err = fmt.Errorf("failed to load index: %w", err)
err = fmt.Errorf("failed to strategically load index: %w", err)
return
}
}
if err = r.LoadFromCache(); err != nil {
err = fmt.Errorf("failed to strategically load index: %w", err)
return return
} }
return return
} }
// LoadFromMemCache attempts to load the Index from the provided cache. // LoadFromPath attempts to load the Index from the configured Path.
// It returns true if the Index was found in the cache, and false otherwise. // It returns an error if no Path is set, or if the load failed.
func (r *ChartRepository) LoadFromMemCache() bool { func (r *ChartRepository) LoadFromPath() error {
if index, found := r.IndexCache.Get(r.IndexKey); found { r.Lock()
r.Lock() defer r.Unlock()
r.Index = index.(*repo.IndexFile)
r.Unlock()
// record the cache hit if len(r.Path) == 0 {
if r.RecordIndexCacheMetric != nil { return fmt.Errorf("no cache path")
r.RecordIndexCacheMetric(cache.CacheEventTypeHit)
}
return true
} }
// record the cache miss i, err := IndexFromFile(r.Path)
if r.RecordIndexCacheMetric != nil { if err != nil {
r.RecordIndexCacheMetric(cache.CacheEventTypeMiss) return fmt.Errorf("failed to load index: %w", err)
} }
return false
}
// LoadFromCache attempts to load the Index from the configured CachePath. r.Index = i
// It returns an error if no CachePath is set, or if the load failed. r.revisions = make(map[digest.Algorithm]digest.Digest, 0)
func (r *ChartRepository) LoadFromCache() error { return nil
if cachePath := r.CachePath; cachePath != "" {
return r.LoadFromFile(cachePath)
}
return fmt.Errorf("no cache path set")
} }
// DownloadIndex attempts to download the chart repository index using // DownloadIndex attempts to download the chart repository index using
// the Client and set Options, and writes the index to the given io.Writer. // the Client and set Options, and writes the index to the given io.Writer.
// It returns an url.Error if the URL failed to parse. // It returns an url.Error if the URL failed to parse.
func (r *ChartRepository) DownloadIndex(w io.Writer) (err error) { func (r *ChartRepository) DownloadIndex(w io.Writer) (err error) {
r.RLock()
defer r.RUnlock()
u, err := url.Parse(r.URL) u, err := url.Parse(r.URL)
if err != nil { if err != nil {
return err return err
@ -460,77 +400,98 @@ func (r *ChartRepository) DownloadIndex(w io.Writer) (err error) {
return nil return nil
} }
// Revision returns the revision of the ChartRepository's Index. It assumes
// the Index is stable sorted.
func (r *ChartRepository) Revision(algorithm digest.Algorithm) digest.Digest {
if !r.HasIndex() {
return ""
}
r.Lock()
defer r.Unlock()
if _, ok := r.revisions[algorithm]; !ok {
if b, _ := yaml.Marshal(r.Index); len(b) > 0 {
r.revisions[algorithm] = algorithm.FromBytes(b)
}
}
return r.revisions[algorithm]
}
// Digest returns the digest of the file at the ChartRepository's Path.
func (r *ChartRepository) Digest(algorithm digest.Algorithm) digest.Digest {
if !r.HasFile() {
return ""
}
r.Lock()
defer r.Unlock()
if _, ok := r.digests[algorithm]; !ok {
if f, err := os.Open(r.Path); err == nil {
defer f.Close()
rd := io.LimitReader(f, helm.MaxIndexSize)
if d, err := algorithm.FromReader(rd); err == nil {
r.digests[algorithm] = d
}
}
}
return r.digests[algorithm]
}
// HasIndex returns true if the Index is not nil. // HasIndex returns true if the Index is not nil.
func (r *ChartRepository) HasIndex() bool { func (r *ChartRepository) HasIndex() bool {
r.RLock() r.RLock()
defer r.RUnlock() defer r.RUnlock()
return r.Index != nil return r.Index != nil
} }
// HasCacheFile returns true if CachePath is not empty. // HasFile returns true if Path exists and is a regular file.
func (r *ChartRepository) HasCacheFile() bool { func (r *ChartRepository) HasFile() bool {
r.RLock() r.RLock()
defer r.RUnlock() defer r.RUnlock()
return r.CachePath != ""
}
// Unload can be used to signal the Go garbage collector the Index can if r.Path != "" {
// be freed from memory if the ChartRepository object is expected to if stat, err := os.Lstat(r.Path); err == nil {
// continue to exist in the stack for some time. return stat.Mode().IsRegular()
func (r *ChartRepository) Unload() {
if r == nil {
return
}
r.Lock()
defer r.Unlock()
r.Index = nil
}
// Clear caches the index in memory before unloading it.
// It cleans up temporary files and directories created by the repository.
func (r *ChartRepository) Clear() error {
var errs []error
if err := r.CacheIndexInMemory(); err != nil {
errs = append(errs, err)
}
r.Unload()
if err := r.RemoveCache(); err != nil {
errs = append(errs, err)
}
return kerrors.NewAggregate(errs)
}
// SetMemCache sets the cache to use for this repository.
func (r *ChartRepository) SetMemCache(key string, c *cache.Cache, ttl time.Duration, rec RecordMetricsFunc) {
r.IndexKey = key
r.IndexCache = c
r.IndexTTL = ttl
r.RecordIndexCacheMetric = rec
}
// RemoveCache removes the CachePath if Cached.
func (r *ChartRepository) RemoveCache() error {
if r == nil {
return nil
}
r.Lock()
defer r.Unlock()
if r.Cached {
if err := os.Remove(r.CachePath); err != nil && !os.IsNotExist(err) {
return err
} }
r.CachePath = ""
r.Cached = false
} }
return false
}
// Clear clears the Index and removes the file at Path, if cached.
func (r *ChartRepository) Clear() error {
r.Lock()
defer r.Unlock()
r.Index = nil
if r.cached {
if err := os.Remove(r.Path); err != nil {
return fmt.Errorf("failed to remove cached index: %w", err)
}
r.Path = ""
r.cached = false
}
r.invalidate()
return nil return nil
} }
// Invalidate clears any cached digests and revisions.
func (r *ChartRepository) Invalidate() {
r.Lock()
defer r.Unlock()
r.invalidate()
}
func (r *ChartRepository) invalidate() {
r.digests = make(map[digest.Algorithm]digest.Digest, 0)
r.revisions = make(map[digest.Algorithm]digest.Digest, 0)
}
// VerifyChart verifies the chart against a signature. // VerifyChart verifies the chart against a signature.
// It returns an error on failure. // It returns an error on failure.
func (r *ChartRepository) VerifyChart(_ context.Context, _ *repo.ChartVersion) error { func (r *ChartRepository) VerifyChart(_ context.Context, _ *repo.ChartVersion) error {

673
internal/helm/repository/chart_repository_test.go
View File

@ -18,20 +18,22 @@ package repository
import ( import (
"bytes" "bytes"
"crypto/sha256" "errors"
"fmt" "fmt"
"net/url" "net/url"
"os" "os"
"path/filepath" "path/filepath"
"sync"
"testing" "testing"
"time" "time"
"github.com/fluxcd/source-controller/internal/cache"
"github.com/fluxcd/source-controller/internal/helm"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
digestlib "github.com/opencontainers/go-digest"
"helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart"
helmgetter "helm.sh/helm/v3/pkg/getter" helmgetter "helm.sh/helm/v3/pkg/getter"
"helm.sh/helm/v3/pkg/repo" "helm.sh/helm/v3/pkg/repo"
"github.com/fluxcd/source-controller/internal/helm"
) )
var now = time.Now() var now = time.Now()
@ -55,6 +57,136 @@ func (g *mockGetter) Get(u string, _ ...helmgetter.Option) (*bytes.Buffer, error
return bytes.NewBuffer(r), nil return bytes.NewBuffer(r), nil
} }
// Index load tests are derived from https://github.com/helm/helm/blob/v3.3.4/pkg/repo/index_test.go#L108
// to ensure parity with Helm behaviour.
func TestIndexFromFile(t *testing.T) {
g := NewWithT(t)
// Create an index file that exceeds the max index size.
tmpDir := t.TempDir()
bigIndexFile := filepath.Join(tmpDir, "index.yaml")
data := make([]byte, helm.MaxIndexSize+10)
g.Expect(os.WriteFile(bigIndexFile, data, 0o640)).ToNot(HaveOccurred())
tests := []struct {
name string
filename string
wantErr string
}{
{
name: "regular index file",
filename: testFile,
},
{
name: "chartmuseum index file",
filename: chartmuseumTestFile,
},
{
name: "error if index size exceeds max size",
filename: bigIndexFile,
wantErr: "exceeds the maximum index file size",
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
i, err := IndexFromFile(tt.filename)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
return
}
g.Expect(err).ToNot(HaveOccurred())
verifyLocalIndex(t, i)
})
}
}
func TestIndexFromBytes(t *testing.T) {
tests := []struct {
name string
b []byte
wantName string
wantVersion string
wantDigest string
wantErr string
}{
{
name: "index",
b: []byte(`
apiVersion: v1
entries:
nginx:
- urls:
- https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz
name: nginx
description: string
version: 0.2.0
home: https://github.com/something/else
digest: "sha256:1234567890abcdef"
`),
wantName: "nginx",
wantVersion: "0.2.0",
wantDigest: "sha256:1234567890abcdef",
},
{
name: "index without API version",
b: []byte(`entries:
nginx:
- name: nginx`),
wantErr: "no API version specified",
},
{
name: "index with duplicate entry",
b: []byte(`apiVersion: v1
entries:
nginx:
- name: nginx"
nginx:
- name: nginx`),
wantErr: "key \"nginx\" already set in map",
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
t.Parallel()
i, err := IndexFromBytes(tt.b)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
g.Expect(i).To(BeNil())
return
}
g.Expect(err).ToNot(HaveOccurred())
g.Expect(i).ToNot(BeNil())
got, err := i.Get(tt.wantName, tt.wantVersion)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got.Digest).To(Equal(tt.wantDigest))
})
}
}
func TestIndexFromBytes_Unordered(t *testing.T) {
b, err := os.ReadFile(unorderedTestFile)
if err != nil {
t.Fatal(err)
}
i, err := IndexFromBytes(b)
if err != nil {
t.Fatal(err)
}
verifyLocalIndex(t, i)
}
func TestNewChartRepository(t *testing.T) { func TestNewChartRepository(t *testing.T) {
repositoryURL := "https://example.com" repositoryURL := "https://example.com"
providers := helmgetter.Providers{ providers := helmgetter.Providers{
@ -68,7 +200,7 @@ func TestNewChartRepository(t *testing.T) {
t.Run("should construct chart repository", func(t *testing.T) { t.Run("should construct chart repository", func(t *testing.T) {
g := NewWithT(t) g := NewWithT(t)
r, err := NewChartRepository(repositoryURL, "", providers, nil, options) r, err := NewChartRepository(repositoryURL, "", providers, nil, options...)
g.Expect(err).ToNot(HaveOccurred()) g.Expect(err).ToNot(HaveOccurred())
g.Expect(r).ToNot(BeNil()) g.Expect(r).ToNot(BeNil())
g.Expect(r.URL).To(Equal(repositoryURL)) g.Expect(r.URL).To(Equal(repositoryURL))
@ -95,7 +227,7 @@ func TestNewChartRepository(t *testing.T) {
}) })
} }
func TestChartRepository_Get(t *testing.T) { func TestChartRepository_GetChartVersion(t *testing.T) {
g := NewWithT(t) g := NewWithT(t)
r := newChartRepository() r := newChartRepository()
@ -252,6 +384,31 @@ func TestChartRepository_DownloadChart(t *testing.T) {
} }
} }
func TestChartRepository_CacheIndex(t *testing.T) {
g := NewWithT(t)
mg := mockGetter{Response: []byte("foo")}
r := newChartRepository()
r.URL = "https://example.com"
r.Client = &mg
r.revisions["key"] = "value"
r.digests["key"] = "value"
err := r.CacheIndex()
g.Expect(err).To(Not(HaveOccurred()))
g.Expect(r.Path).ToNot(BeEmpty())
t.Cleanup(func() { _ = os.Remove(r.Path) })
g.Expect(r.Path).To(BeARegularFile())
b, _ := os.ReadFile(r.Path)
g.Expect(b).To(Equal(mg.Response))
g.Expect(r.revisions).To(BeEmpty())
g.Expect(r.digests).To(BeEmpty())
}
func TestChartRepository_DownloadIndex(t *testing.T) { func TestChartRepository_DownloadIndex(t *testing.T) {
g := NewWithT(t) g := NewWithT(t)
@ -260,8 +417,9 @@ func TestChartRepository_DownloadIndex(t *testing.T) {
mg := mockGetter{Response: b} mg := mockGetter{Response: b}
r := &ChartRepository{ r := &ChartRepository{
URL: "https://example.com", URL: "https://example.com",
Client: &mg, Client: &mg,
RWMutex: &sync.RWMutex{},
} }
buf := bytes.NewBuffer([]byte{}) buf := bytes.NewBuffer([]byte{})
@ -271,258 +429,166 @@ func TestChartRepository_DownloadIndex(t *testing.T) {
g.Expect(err).To(BeNil()) g.Expect(err).To(BeNil())
} }
func TestChartRepository_LoadIndexFromBytes(t *testing.T) {
tests := []struct {
name string
b []byte
wantName string
wantVersion string
wantDigest string
wantErr string
}{
{
name: "index",
b: []byte(`
apiVersion: v1
entries:
nginx:
- urls:
- https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz
name: nginx
description: string
version: 0.2.0
home: https://github.com/something/else
digest: "sha256:1234567890abcdef"
`),
wantName: "nginx",
wantVersion: "0.2.0",
wantDigest: "sha256:1234567890abcdef",
},
{
name: "index without API version",
b: []byte(`entries:
nginx:
- name: nginx`),
wantErr: "no API version specified",
},
{
name: "index with duplicate entry",
b: []byte(`apiVersion: v1
entries:
nginx:
- name: nginx"
nginx:
- name: nginx`),
wantErr: "key \"nginx\" already set in map",
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
t.Parallel()
r := newChartRepository()
err := r.LoadIndexFromBytes(tt.b)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
g.Expect(r.Index).To(BeNil())
return
}
g.Expect(err).ToNot(HaveOccurred())
g.Expect(r.Index).ToNot(BeNil())
got, err := r.Index.Get(tt.wantName, tt.wantVersion)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got.Digest).To(Equal(tt.wantDigest))
})
}
}
func TestChartRepository_LoadIndexFromBytes_Unordered(t *testing.T) {
b, err := os.ReadFile(unorderedTestFile)
if err != nil {
t.Fatal(err)
}
r := newChartRepository()
err = r.LoadIndexFromBytes(b)
if err != nil {
t.Fatal(err)
}
verifyLocalIndex(t, r.Index)
}
// Index load tests are derived from https://github.com/helm/helm/blob/v3.3.4/pkg/repo/index_test.go#L108
// to ensure parity with Helm behaviour.
func TestChartRepository_LoadIndexFromFile(t *testing.T) {
g := NewWithT(t)
// Create an index file that exceeds the max index size.
tmpDir := t.TempDir()
bigIndexFile := filepath.Join(tmpDir, "index.yaml")
data := make([]byte, helm.MaxIndexSize+10)
g.Expect(os.WriteFile(bigIndexFile, data, 0o640)).ToNot(HaveOccurred())
tests := []struct {
name string
filename string
wantErr string
}{
{
name: "regular index file",
filename: testFile,
},
{
name: "chartmuseum index file",
filename: chartmuseumTestFile,
},
{
name: "error if index size exceeds max size",
filename: bigIndexFile,
wantErr: "size of index 'index.yaml' exceeds",
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
r := newChartRepository()
err := r.LoadFromFile(tt.filename)
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
return
}
g.Expect(err).ToNot(HaveOccurred())
verifyLocalIndex(t, r.Index)
})
}
}
func TestChartRepository_CacheIndex(t *testing.T) {
g := NewWithT(t)
mg := mockGetter{Response: []byte("foo")}
expectSum := fmt.Sprintf("%x", sha256.Sum256(mg.Response))
r := newChartRepository()
r.URL = "https://example.com"
r.Client = &mg
sum, err := r.CacheIndex()
g.Expect(err).To(Not(HaveOccurred()))
g.Expect(r.CachePath).ToNot(BeEmpty())
defer os.RemoveAll(r.CachePath)
g.Expect(r.CachePath).To(BeARegularFile())
b, _ := os.ReadFile(r.CachePath)
g.Expect(b).To(Equal(mg.Response))
g.Expect(sum).To(BeEquivalentTo(expectSum))
}
func TestChartRepository_StrategicallyLoadIndex(t *testing.T) { func TestChartRepository_StrategicallyLoadIndex(t *testing.T) {
g := NewWithT(t) t.Run("loads from path", func(t *testing.T) {
g := NewWithT(t)
r := newChartRepository() i := filepath.Join(t.TempDir(), "index.yaml")
r.Index = repo.NewIndexFile() g.Expect(os.WriteFile(i, []byte(`apiVersion: v1`), 0o644)).To(Succeed())
g.Expect(r.StrategicallyLoadIndex()).To(Succeed())
g.Expect(r.CachePath).To(BeEmpty())
g.Expect(r.Cached).To(BeFalse())
r.Index = nil r := newChartRepository()
r.CachePath = "/invalid/cache/index/path.yaml" r.Path = i
err := r.StrategicallyLoadIndex()
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring("/invalid/cache/index/path.yaml: no such file or directory"))
g.Expect(r.Cached).To(BeFalse())
r.CachePath = "" err := r.StrategicallyLoadIndex()
r.Client = &mockGetter{} g.Expect(err).To(Succeed())
err = r.StrategicallyLoadIndex() g.Expect(r.Index).ToNot(BeNil())
g.Expect(err).To(HaveOccurred()) })
g.Expect(err.Error()).To(ContainSubstring("no API version specified"))
g.Expect(r.Cached).To(BeTrue()) t.Run("loads from client", func(t *testing.T) {
g.Expect(r.RemoveCache()).To(Succeed()) g := NewWithT(t)
}
r := newChartRepository()
func TestChartRepository_CacheIndexInMemory(t *testing.T) { r.Client = &mockGetter{
g := NewWithT(t) Response: []byte(`apiVersion: v1`),
}
interval, _ := time.ParseDuration("5s") t.Cleanup(func() {
memCache := cache.New(1, interval) _ = os.Remove(r.Path)
indexPath := "/multi-tenent-safe/mock/index.yaml" })
r := newChartRepository()
r.Index = repo.NewIndexFile() err := r.StrategicallyLoadIndex()
indexFile := *r.Index g.Expect(err).To(Succeed())
g.Expect( g.Expect(r.Path).ToNot(BeEmpty())
indexFile.MustAdd( g.Expect(r.Index).ToNot(BeNil())
&chart.Metadata{ })
Name: "grafana",
Version: "6.17.4", t.Run("skips if index is already loaded", func(t *testing.T) {
}, g := NewWithT(t)
"grafana-6.17.4.tgz",
"http://example.com/charts", r := newChartRepository()
"sha256:1234567890abc", r.Index = repo.NewIndexFile()
)).To(Succeed())
indexFile.WriteFile(indexPath, 0o640) g.Expect(r.StrategicallyLoadIndex()).To(Succeed())
ttl, _ := time.ParseDuration("1m")
r.SetMemCache(indexPath, memCache, ttl, func(event string) {
fmt.Println(event)
}) })
r.CacheIndexInMemory()
_, cacheHit := r.IndexCache.Get(indexPath)
g.Expect(cacheHit).To(Equal(true))
r.Unload()
g.Expect(r.Index).To(BeNil())
g.Expect(r.StrategicallyLoadIndex()).To(Succeed())
g.Expect(r.Index.Entries["grafana"][0].Digest).To(Equal("sha256:1234567890abc"))
} }
func TestChartRepository_LoadFromCache(t *testing.T) { func TestChartRepository_LoadFromPath(t *testing.T) {
tests := []struct { t.Run("loads index", func(t *testing.T) {
name string g := NewWithT(t)
cachePath string
wantErr string
}{
{
name: "cache path",
cachePath: chartmuseumTestFile,
},
{
name: "invalid cache path",
cachePath: "invalid",
wantErr: "stat invalid: no such file",
},
{
name: "no cache path",
cachePath: "",
wantErr: "no cache path set",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
r := newChartRepository() i := filepath.Join(t.TempDir(), "index.yaml")
r.CachePath = tt.cachePath g.Expect(os.WriteFile(i, []byte(`apiVersion: v1`), 0o644)).To(Succeed())
err := r.LoadFromCache()
if tt.wantErr != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring(tt.wantErr))
g.Expect(r.Index).To(BeNil())
return
}
g.Expect(err).ToNot(HaveOccurred()) r := newChartRepository()
verifyLocalIndex(t, r.Index) r.Path = i
}) r.revisions["key"] = "value"
}
g.Expect(r.LoadFromPath()).To(Succeed())
g.Expect(r.Index).ToNot(BeNil())
g.Expect(r.revisions).To(BeEmpty())
})
t.Run("no cache path", func(t *testing.T) {
g := NewWithT(t)
err := newChartRepository().LoadFromPath()
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(ContainSubstring("no cache path"))
})
t.Run("index load error", func(t *testing.T) {
g := NewWithT(t)
r := newChartRepository()
r.Path = filepath.Join(t.TempDir(), "index.yaml")
err := r.LoadFromPath()
g.Expect(err).To(HaveOccurred())
g.Expect(errors.Is(err, os.ErrNotExist)).To(BeTrue())
})
}
func TestChartRepository_Revision(t *testing.T) {
t.Run("with algorithm", func(t *testing.T) {
r := newChartRepository()
r.Index = repo.NewIndexFile()
for _, algo := range []digestlib.Algorithm{digestlib.SHA256, digestlib.SHA512} {
t.Run(algo.String(), func(t *testing.T) {
g := NewWithT(t)
d := r.Revision(algo)
g.Expect(d).ToNot(BeEmpty())
g.Expect(d.Algorithm()).To(Equal(algo))
g.Expect(r.revisions[algo]).To(Equal(d))
})
}
})
t.Run("without index", func(t *testing.T) {
g := NewWithT(t)
r := newChartRepository()
g.Expect(r.Revision(digestlib.SHA256)).To(BeEmpty())
})
t.Run("from cache", func(t *testing.T) {
g := NewWithT(t)
algo := digestlib.SHA256
expect := digestlib.Digest("sha256:fake")
r := newChartRepository()
r.Index = repo.NewIndexFile()
r.revisions[algo] = expect
g.Expect(r.Revision(algo)).To(Equal(expect))
})
}
func TestChartRepository_Digest(t *testing.T) {
t.Run("with algorithm", func(t *testing.T) {
g := NewWithT(t)
p := filepath.Join(t.TempDir(), "index.yaml")
g.Expect(repo.NewIndexFile().WriteFile(p, 0o644)).To(Succeed())
r := newChartRepository()
r.Path = p
for _, algo := range []digestlib.Algorithm{digestlib.SHA256, digestlib.SHA512} {
t.Run(algo.String(), func(t *testing.T) {
g := NewWithT(t)
d := r.Digest(algo)
g.Expect(d).ToNot(BeEmpty())
g.Expect(d.Algorithm()).To(Equal(algo))
g.Expect(r.digests[algo]).To(Equal(d))
})
}
})
t.Run("without path", func(t *testing.T) {
g := NewWithT(t)
r := newChartRepository()
g.Expect(r.Digest(digestlib.SHA256)).To(BeEmpty())
})
t.Run("from cache", func(t *testing.T) {
g := NewWithT(t)
algo := digestlib.SHA256
expect := digestlib.Digest("sha256:fake")
i := filepath.Join(t.TempDir(), "index.yaml")
g.Expect(os.WriteFile(i, []byte(`apiVersion: v1`), 0o644)).To(Succeed())
r := newChartRepository()
r.Path = i
r.digests[algo] = expect
g.Expect(r.Digest(algo)).To(Equal(expect))
})
} }
func TestChartRepository_HasIndex(t *testing.T) { func TestChartRepository_HasIndex(t *testing.T) {
@ -534,23 +600,88 @@ func TestChartRepository_HasIndex(t *testing.T) {
g.Expect(r.HasIndex()).To(BeTrue()) g.Expect(r.HasIndex()).To(BeTrue())
} }
func TestChartRepository_HasCacheFile(t *testing.T) { func TestChartRepository_HasFile(t *testing.T) {
g := NewWithT(t) g := NewWithT(t)
r := newChartRepository() r := newChartRepository()
g.Expect(r.HasCacheFile()).To(BeFalse()) g.Expect(r.HasFile()).To(BeFalse())
r.CachePath = "foo"
g.Expect(r.HasCacheFile()).To(BeTrue()) i := filepath.Join(t.TempDir(), "index.yaml")
g.Expect(os.WriteFile(i, []byte(`apiVersion: v1`), 0o644)).To(Succeed())
r.Path = i
g.Expect(r.HasFile()).To(BeTrue())
} }
func TestChartRepository_UnloadIndex(t *testing.T) { func TestChartRepository_Clear(t *testing.T) {
t.Run("without index", func(t *testing.T) {
g := NewWithT(t)
r := newChartRepository()
g.Expect(r.Clear()).To(Succeed())
})
t.Run("with index", func(t *testing.T) {
g := NewWithT(t)
r := newChartRepository()
r.Index = repo.NewIndexFile()
r.revisions["key"] = "value"
g.Expect(r.Clear()).To(Succeed())
g.Expect(r.Index).To(BeNil())
g.Expect(r.revisions).To(BeEmpty())
})
t.Run("with index and cached path", func(t *testing.T) {
g := NewWithT(t)
f, err := os.CreateTemp(t.TempDir(), "index-*.yaml")
g.Expect(err).ToNot(HaveOccurred())
g.Expect(f.Close()).To(Succeed())
r := newChartRepository()
r.Path = f.Name()
r.Index = repo.NewIndexFile()
r.digests["key"] = "value"
r.revisions["key"] = "value"
r.cached = true
g.Expect(r.Clear()).To(Succeed())
g.Expect(r.Index).To(BeNil())
g.Expect(r.Path).To(BeEmpty())
g.Expect(r.digests).To(BeEmpty())
g.Expect(r.revisions).To(BeEmpty())
g.Expect(r.cached).To(BeFalse())
})
t.Run("with path", func(t *testing.T) {
g := NewWithT(t)
f, err := os.CreateTemp(t.TempDir(), "index-*.yaml")
g.Expect(err).ToNot(HaveOccurred())
g.Expect(f.Close()).To(Succeed())
r := newChartRepository()
r.Path = f.Name()
r.digests["key"] = "value"
g.Expect(r.Clear()).To(Succeed())
g.Expect(r.Path).ToNot(BeEmpty())
g.Expect(r.Path).To(BeARegularFile())
g.Expect(r.digests).To(BeEmpty())
})
}
func TestChartRepository_Invalidate(t *testing.T) {
g := NewWithT(t) g := NewWithT(t)
r := newChartRepository() r := newChartRepository()
g.Expect(r.HasIndex()).To(BeFalse()) r.digests["key"] = "value"
r.Index = repo.NewIndexFile() r.revisions["key"] = "value"
r.Unload()
g.Expect(r.Index).To(BeNil()) r.Invalidate()
g.Expect(r.digests).To(BeEmpty())
g.Expect(r.revisions).To(BeEmpty())
} }
func verifyLocalIndex(t *testing.T, i *repo.IndexFile) { func verifyLocalIndex(t *testing.T, i *repo.IndexFile) {
@ -622,27 +753,3 @@ func verifyLocalIndex(t *testing.T, i *repo.IndexFile) {
g.Expect(tt.Keywords).To(ContainElements(expect.Keywords)) g.Expect(tt.Keywords).To(ContainElements(expect.Keywords))
} }
} }
func TestChartRepository_RemoveCache(t *testing.T) {
g := NewWithT(t)
tmpFile, err := os.CreateTemp("", "remove-cache-")
g.Expect(err).ToNot(HaveOccurred())
defer os.Remove(tmpFile.Name())
r := newChartRepository()
r.CachePath = tmpFile.Name()
r.Cached = true
g.Expect(r.RemoveCache()).To(Succeed())
g.Expect(r.CachePath).To(BeEmpty())
g.Expect(r.Cached).To(BeFalse())
g.Expect(tmpFile.Name()).ToNot(BeAnExistingFile())
r.CachePath = tmpFile.Name()
r.Cached = true
g.Expect(r.RemoveCache()).To(Succeed())
g.Expect(r.CachePath).To(BeEmpty())
g.Expect(r.Cached).To(BeFalse())
}