diff --git a/api/v1beta2/gitrepository_types.go b/api/v1beta2/gitrepository_types.go index 0ecf2ba5..6ab6f369 100644 --- a/api/v1beta2/gitrepository_types.go +++ b/api/v1beta2/gitrepository_types.go @@ -19,12 +19,10 @@ package v1beta2 import ( "time" - apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fluxcd/pkg/apis/acl" "github.com/fluxcd/pkg/apis/meta" - "github.com/fluxcd/pkg/runtime/conditions" ) const ( @@ -202,48 +200,6 @@ const ( GitOperationFailedReason string = "GitOperationFailed" ) -// GitRepositoryProgressing resets the conditions of the GitRepository to -// metav1.Condition of type meta.ReadyCondition with status 'Unknown' and -// meta.ProgressingReason reason and message. It returns the modified -// GitRepository. -func GitRepositoryProgressing(repository GitRepository) GitRepository { - repository.Status.ObservedGeneration = repository.Generation - repository.Status.URL = "" - repository.Status.Conditions = []metav1.Condition{} - conditions.MarkUnknown(&repository, meta.ReadyCondition, meta.ProgressingReason, "reconciliation in progress") - return repository -} - -// GitRepositoryReady sets the given Artifact and URL on the GitRepository and -// sets the meta.ReadyCondition to 'True', with the given reason and message. It -// returns the modified GitRepository. -func GitRepositoryReady(repository GitRepository, artifact Artifact, includedArtifacts []*Artifact, url, reason, message string) GitRepository { - repository.Status.Artifact = &artifact - repository.Status.IncludedArtifacts = includedArtifacts - repository.Status.URL = url - conditions.MarkTrue(&repository, meta.ReadyCondition, reason, message) - return repository -} - -// GitRepositoryNotReady sets the meta.ReadyCondition on the given GitRepository -// to 'False', with the given reason and message. It returns the modified -// GitRepository. -func GitRepositoryNotReady(repository GitRepository, reason, message string) GitRepository { - conditions.MarkFalse(&repository, meta.ReadyCondition, reason, message) - return repository -} - -// GitRepositoryReadyMessage returns the message of the metav1.Condition of type -// meta.ReadyCondition with status 'True' if present, or an empty string. -func GitRepositoryReadyMessage(repository GitRepository) string { - if c := apimeta.FindStatusCondition(repository.Status.Conditions, meta.ReadyCondition); c != nil { - if c.Status == metav1.ConditionTrue { - return c.Message - } - } - return "" -} - // GetConditions returns the status conditions of the object. func (in GitRepository) GetConditions() []metav1.Condition { return in.Status.Conditions diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index 976b24c0..9b88bc01 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -20,18 +20,15 @@ import ( "context" "fmt" "os" - "path/filepath" "strings" "time" securejoin "github.com/cyphar/filepath-securejoin" corev1 "k8s.io/api/core/v1" - apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + kerrors "k8s.io/apimachinery/pkg/util/errors" kuberecorder "k8s.io/client-go/tools/record" - "k8s.io/client-go/tools/reference" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -40,14 +37,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/runtime/conditions" + helper "github.com/fluxcd/pkg/runtime/controller" "github.com/fluxcd/pkg/runtime/events" - "github.com/fluxcd/pkg/runtime/metrics" + "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/predicates" + "github.com/fluxcd/source-controller/pkg/sourceignore" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" "github.com/fluxcd/source-controller/pkg/git" "github.com/fluxcd/source-controller/pkg/git/strategy" - "github.com/fluxcd/source-controller/pkg/sourceignore" ) // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories,verbs=get;list;watch;create;update;patch;delete @@ -58,12 +57,12 @@ import ( // GitRepositoryReconciler reconciles a GitRepository object type GitRepositoryReconciler struct { client.Client - requeueDependency time.Duration - Scheme *runtime.Scheme - Storage *Storage - EventRecorder kuberecorder.EventRecorder - ExternalEventRecorder *events.Recorder - MetricsRecorder *metrics.Recorder + kuberecorder.EventRecorder + helper.Metrics + + Storage *Storage + + requeueDependency time.Duration } type GitRepositoryReconcilerOptions struct { @@ -86,398 +85,503 @@ func (r *GitRepositoryReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, o Complete(r) } -func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { +func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, retErr error) { start := time.Now() log := ctrl.LoggerFrom(ctx) - var repository sourcev1.GitRepository - if err := r.Get(ctx, req.NamespacedName, &repository); err != nil { + // Fetch the GitRepository + obj := &sourcev1.GitRepository{} + if err := r.Get(ctx, req.NamespacedName, obj); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } // Record suspended status metric - defer r.recordSuspension(ctx, repository) + r.RecordSuspend(ctx, obj, obj.Spec.Suspend) - // Add our finalizer if it does not exist - if !controllerutil.ContainsFinalizer(&repository, sourcev1.SourceFinalizer) { - patch := client.MergeFrom(repository.DeepCopy()) - controllerutil.AddFinalizer(&repository, sourcev1.SourceFinalizer) - if err := r.Patch(ctx, &repository, patch); err != nil { - log.Error(err, "unable to register finalizer") - return ctrl.Result{}, err - } - } - - // Examine if the object is under deletion - if !repository.ObjectMeta.DeletionTimestamp.IsZero() { - return r.reconcileDelete(ctx, repository) - } - - // Return early if the object is suspended. - if repository.Spec.Suspend { + // Return early if the object is suspended + if obj.Spec.Suspend { log.Info("Reconciliation is suspended for this object") return ctrl.Result{}, nil } - // check dependencies - if len(repository.Spec.Include) > 0 { - if err := r.checkDependencies(repository); err != nil { - repository = sourcev1.GitRepositoryNotReady(repository, "DependencyNotReady", err.Error()) - if err := r.updateStatus(ctx, req, repository.Status); err != nil { - log.Error(err, "unable to update status for dependency not ready") - return ctrl.Result{Requeue: true}, err - } - // we can't rely on exponential backoff because it will prolong the execution too much, - // instead we requeue on a fix interval. - msg := fmt.Sprintf("Dependencies do not meet ready condition, retrying in %s", r.requeueDependency.String()) - log.Info(msg) - r.event(ctx, repository, events.EventSeverityInfo, msg) - r.recordReadiness(ctx, repository) - return ctrl.Result{RequeueAfter: r.requeueDependency}, nil - } - log.Info("All dependencies area ready, proceeding with reconciliation") - } - - // record reconciliation duration - if r.MetricsRecorder != nil { - objRef, err := reference.GetReference(r.Scheme, &repository) - if err != nil { - return ctrl.Result{}, err - } - defer r.MetricsRecorder.RecordDuration(*objRef, start) - } - - // set initial status - if resetRepository, ok := r.resetStatus(repository); ok { - repository = resetRepository - if err := r.updateStatus(ctx, req, repository.Status); err != nil { - log.Error(err, "unable to update status") - return ctrl.Result{Requeue: true}, err - } - r.recordReadiness(ctx, repository) - } - - // record the value of the reconciliation request, if any - // TODO(hidde): would be better to defer this in combination with - // always patching the status sub-resource after a reconciliation. - if v, ok := meta.ReconcileAnnotationValue(repository.GetAnnotations()); ok { - repository.Status.SetLastHandledReconcileRequest(v) - } - - // purge old artifacts from storage - if err := r.gc(repository); err != nil { - log.Error(err, "unable to purge old artifacts") - } - - // reconcile repository by pulling the latest Git commit - reconciledRepository, reconcileErr := r.reconcile(ctx, *repository.DeepCopy()) - - // update status with the reconciliation result - if err := r.updateStatus(ctx, req, reconciledRepository.Status); err != nil { - log.Error(err, "unable to update status") - return ctrl.Result{Requeue: true}, err - } - - // if reconciliation failed, record the failure and requeue immediately - if reconcileErr != nil { - r.event(ctx, reconciledRepository, events.EventSeverityError, reconcileErr.Error()) - r.recordReadiness(ctx, reconciledRepository) - return ctrl.Result{Requeue: true}, reconcileErr - } - - // emit revision change event - if repository.Status.Artifact == nil || reconciledRepository.Status.Artifact.Revision != repository.Status.Artifact.Revision { - r.event(ctx, reconciledRepository, events.EventSeverityInfo, sourcev1.GitRepositoryReadyMessage(reconciledRepository)) - } - r.recordReadiness(ctx, reconciledRepository) - - log.Info(fmt.Sprintf("Reconciliation finished in %s, next run in %s", - time.Since(start).String(), - repository.GetInterval().Duration.String(), - )) - - return ctrl.Result{RequeueAfter: repository.GetInterval().Duration}, nil -} - -func (r *GitRepositoryReconciler) checkDependencies(repository sourcev1.GitRepository) error { - for _, d := range repository.Spec.Include { - dName := types.NamespacedName{Name: d.GitRepositoryRef.Name, Namespace: repository.Namespace} - var gr sourcev1.GitRepository - err := r.Get(context.Background(), dName, &gr) - if err != nil { - return fmt.Errorf("unable to get '%s' dependency: %w", dName, err) - } - - if len(gr.Status.Conditions) == 0 || gr.Generation != gr.Status.ObservedGeneration { - return fmt.Errorf("dependency '%s' is not ready", dName) - } - - if !apimeta.IsStatusConditionTrue(gr.Status.Conditions, meta.ReadyCondition) { - return fmt.Errorf("dependency '%s' is not ready", dName) - } - } - - return nil -} - -func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sourcev1.GitRepository) (sourcev1.GitRepository, error) { - log := ctrl.LoggerFrom(ctx) - - // create tmp dir for the Git clone - tmpGit, err := os.MkdirTemp("", repository.Name) + // Initialize the patch helper + patchHelper, err := patch.NewHelper(obj, r.Client) if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + return ctrl.Result{}, err } + + // Always attempt to patch the object and status after each reconciliation defer func() { - if err := os.RemoveAll(tmpGit); err != nil { - log.Error(err, "failed to remove working directory", "path", tmpGit) + // Record the value of the reconciliation request, if any + if v, ok := meta.ReconcileAnnotationValue(obj.GetAnnotations()); ok { + obj.Status.SetLastHandledReconcileRequest(v) } + + // Summarize the Ready condition based on abnormalities that may have been observed. + conditions.SetSummary(obj, + meta.ReadyCondition, + conditions.WithConditions( + sourcev1.IncludeUnavailableCondition, + sourcev1.SourceVerifiedCondition, + sourcev1.CheckoutFailedCondition, + sourcev1.ArtifactOutdatedCondition, + sourcev1.ArtifactUnavailableCondition, + ), + conditions.WithNegativePolarityConditions( + sourcev1.ArtifactUnavailableCondition, + sourcev1.CheckoutFailedCondition, + sourcev1.SourceVerifiedCondition, + sourcev1.IncludeUnavailableCondition, + sourcev1.ArtifactOutdatedCondition, + ), + ) + + // Patch the object, ignoring conflicts on the conditions owned by this controller + patchOpts := []patch.Option{ + patch.WithOwnedConditions{ + Conditions: []string{ + sourcev1.ArtifactUnavailableCondition, + sourcev1.CheckoutFailedCondition, + sourcev1.IncludeUnavailableCondition, + sourcev1.ArtifactOutdatedCondition, + meta.ReadyCondition, + meta.ReconcilingCondition, + meta.StalledCondition, + }, + }, + } + + // Determine if the resource is still being reconciled, or if it has stalled, and record this observation + if retErr == nil && (result.IsZero() || !result.Requeue) { + // We are no longer reconciling + conditions.Delete(obj, meta.ReconcilingCondition) + + // We have now observed this generation + patchOpts = append(patchOpts, patch.WithStatusObservedGeneration{}) + + readyCondition := conditions.Get(obj, meta.ReadyCondition) + switch readyCondition.Status { + case metav1.ConditionFalse: + // As we are no longer reconciling and the end-state is not ready, the reconciliation has stalled + conditions.MarkStalled(obj, readyCondition.Reason, readyCondition.Message) + case metav1.ConditionTrue: + // As we are no longer reconciling and the end-state is ready, the reconciliation is no longer stalled + conditions.Delete(obj, meta.StalledCondition) + } + } + + // Finally, patch the resource + if err := patchHelper.Patch(ctx, obj, patchOpts...); err != nil { + retErr = kerrors.NewAggregate([]error{retErr, err}) + } + + // Always record readiness and duration metrics + r.Metrics.RecordReadiness(ctx, obj) + r.Metrics.RecordDuration(ctx, obj, start) }() - // Configure auth options using secret - var authOpts *git.AuthOptions - if repository.Spec.SecretRef != nil { + // Add finalizer first if not exist to avoid the race condition + // between init and delete + if !controllerutil.ContainsFinalizer(obj, sourcev1.SourceFinalizer) { + controllerutil.AddFinalizer(obj, sourcev1.SourceFinalizer) + return ctrl.Result{Requeue: true}, nil + } + + // Examine if the object is under deletion + if !obj.ObjectMeta.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, obj) + } + + // Reconcile actual object + return r.reconcile(ctx, obj) +} + +// reconcile steps through the actual reconciliation tasks for the object, it returns early on the first step that +// produces an error. +func (r *GitRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.GitRepository) (ctrl.Result, error) { + // Mark the resource as under reconciliation + conditions.MarkReconciling(obj, meta.ProgressingReason, "") + + // Reconcile the storage data + if result, err := r.reconcileStorage(ctx, obj); err != nil || result.IsZero() { + return result, err + } + + // Create temp dir for Git clone + tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-%s-", obj.Kind, obj.Namespace, obj.Name)) + if err != nil { + r.Eventf(obj, events.EventSeverityError, sourcev1.StorageOperationFailedReason, "Failed to create temporary directory: %s", err) + return ctrl.Result{}, err + } + defer os.RemoveAll(tmpDir) + + // Reconcile the source from upstream + var artifact sourcev1.Artifact + if result, err := r.reconcileSource(ctx, obj, &artifact, tmpDir); err != nil || result.IsZero() { + return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, err + } + + // Reconcile includes from the storage + var includes artifactSet + if result, err := r.reconcileInclude(ctx, obj, includes, tmpDir); err != nil || result.IsZero() { + return ctrl.Result{RequeueAfter: r.requeueDependency}, err + } + + // Reconcile the artifact to storage + if result, err := r.reconcileArtifact(ctx, obj, artifact, includes, tmpDir); err != nil || result.IsZero() { + return result, err + } + + return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil +} + +// reconcileStorage ensures the current state of the storage matches the desired and previously observed state. +// +// All artifacts for the resource except for the current one are garbage collected from the storage. +// If the artifact in the Status object of the resource disappeared from storage, it is removed from the object. +// If the object does not have an artifact in its Status object, a v1beta1.ArtifactUnavailableCondition is set. +// If the hostname of any of the URLs on the object do not match the current storage server hostname, they are updated. +// +// The caller should assume a failure if an error is returned, or the Result is zero. +func (r *GitRepositoryReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.GitRepository) (ctrl.Result, error) { + // Garbage collect previous advertised artifact(s) from storage + _ = r.garbageCollect(ctx, obj) + + // Determine if the advertised artifact is still in storage + if artifact := obj.GetArtifact(); artifact != nil && !r.Storage.ArtifactExist(*artifact) { + obj.Status.Artifact = nil + obj.Status.URL = "" + } + + // Record that we do not have an artifact + if obj.GetArtifact() == nil { + conditions.MarkTrue(obj, sourcev1.ArtifactUnavailableCondition, "NoArtifact", "No artifact for resource in storage") + return ctrl.Result{Requeue: true}, nil + } + conditions.Delete(obj, sourcev1.ArtifactUnavailableCondition) + + // Always update URLs to ensure hostname is up-to-date + // TODO(hidde): we may want to send out an event only if we notice the URL has changed + r.Storage.SetArtifactURL(obj.GetArtifact()) + obj.Status.URL = r.Storage.SetHostname(obj.Status.URL) + + return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil +} + +// reconcileSource ensures the upstream Git repository can be reached and checked out using the declared configuration, +// and observes its state. +// +// The repository is checked out to the given dir using the defined configuration, and in case of an error during the +// checkout process (including transient errors), it records v1beta1.CheckoutFailedCondition=True and returns early. +// On a successful checkout it removes v1beta1.CheckoutFailedCondition, and compares the current revision of HEAD to the +// artifact on the object, and records v1beta1.ArtifactOutdatedCondition if they differ. +// If instructed, the signature of the commit is verified if and recorded as v1beta1.SourceVerifiedCondition. If the +// signature can not be verified or the verification fails, the Condition=False and it returns early. +// If both the checkout and signature verification are successful, the given artifact pointer is set to a new artifact +// with the available metadata. +// +// The caller should assume a failure if an error is returned, or the Result is zero. +func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context, + obj *sourcev1.GitRepository, artifact *sourcev1.Artifact, dir string) (ctrl.Result, error) { + // Configure authentication strategy to access the source + authOpts := &git.AuthOptions{} + if obj.Spec.SecretRef != nil { + // Attempt to retrieve secret name := types.NamespacedName{ - Namespace: repository.GetNamespace(), - Name: repository.Spec.SecretRef.Name, + Namespace: obj.GetNamespace(), + Name: obj.Spec.SecretRef.Name, + } + var secret corev1.Secret + if err := r.Client.Get(ctx, name, &secret); err != nil { + conditions.MarkTrue(obj, sourcev1.CheckoutFailedCondition, sourcev1.AuthenticationFailedReason, + "Failed to get secret '%s': %s", name.String(), err.Error()) + r.Eventf(obj, events.EventSeverityError, sourcev1.AuthenticationFailedReason, + "Failed to get secret '%s': %s", name.String(), err.Error()) + // Return error as the world as observed may change + return ctrl.Result{}, err } - secret := &corev1.Secret{} - err = r.Client.Get(ctx, name, secret) + // Configure strategy with secret + var err error + authOpts, err = git.AuthOptionsFromSecret(obj.Spec.URL, &secret) if err != nil { - err = fmt.Errorf("auth secret error: %w", err) - return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err - } - - authOpts, err = git.AuthOptionsFromSecret(repository.Spec.URL, secret) - if err != nil { - return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err + conditions.MarkTrue(obj, sourcev1.CheckoutFailedCondition, sourcev1.AuthenticationFailedReason, + "Failed to configure auth strategy for Git implementation %q: %s", obj.Spec.GitImplementation, err) + r.Eventf(obj, events.EventSeverityError, sourcev1.AuthenticationFailedReason, + "Failed to configure auth strategy for Git implementation %q: %s", obj.Spec.GitImplementation, err) + // Return error as the contents of the secret may change + return ctrl.Result{}, err } } - checkoutOpts := git.CheckoutOptions{RecurseSubmodules: repository.Spec.RecurseSubmodules} - if ref := repository.Spec.Reference; ref != nil { + + // Configure checkout strategy + checkoutOpts := git.CheckoutOptions{RecurseSubmodules: obj.Spec.RecurseSubmodules} + if ref := obj.Spec.Reference; ref != nil { checkoutOpts.Branch = ref.Branch checkoutOpts.Commit = ref.Commit checkoutOpts.Tag = ref.Tag checkoutOpts.SemVer = ref.SemVer } checkoutStrategy, err := strategy.CheckoutStrategyForImplementation(ctx, - git.Implementation(repository.Spec.GitImplementation), checkoutOpts) + git.Implementation(obj.Spec.GitImplementation), checkoutOpts) if err != nil { - return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err + conditions.MarkTrue(obj, sourcev1.CheckoutFailedCondition, sourcev1.GitOperationFailedReason, + "Failed to configure checkout strategy for Git implementation %q: %s", obj.Spec.GitImplementation, err) + // Do not return err as recovery without changes is impossible + return ctrl.Result{}, nil } - gitCtx, cancel := context.WithTimeout(ctx, repository.Spec.Timeout.Duration) + // Checkout HEAD of reference in object + gitCtx, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration) defer cancel() - - commit, err := checkoutStrategy.Checkout(gitCtx, tmpGit, repository.Spec.URL, authOpts) + commit, err := checkoutStrategy.Checkout(gitCtx, dir, obj.Spec.URL, authOpts) if err != nil { - return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err + conditions.MarkTrue(obj, sourcev1.CheckoutFailedCondition, sourcev1.GitOperationFailedReason, + "Failed to checkout and determine revision: %s", err) + r.Eventf(obj, events.EventSeverityError, sourcev1.GitOperationFailedReason, + "Failed to checkout and determine revision: %s", err) + // Coin flip on transient or persistent error, return error and hope for the best + return ctrl.Result{}, err } - artifact := r.Storage.NewArtifactFor(repository.Kind, repository.GetObjectMeta(), commit.String(), fmt.Sprintf("%s.tar.gz", commit.Hash.String())) + r.Eventf(obj, events.EventSeverityInfo, sourcev1.GitOperationSucceedReason, + "Cloned repository '%s' and checked out revision '%s'", obj.Spec.URL, commit.String()) + conditions.Delete(obj, sourcev1.CheckoutFailedCondition) - // copy all included repository into the artifact - includedArtifacts := []*sourcev1.Artifact{} - for _, incl := range repository.Spec.Include { - dName := types.NamespacedName{Name: incl.GitRepositoryRef.Name, Namespace: repository.Namespace} - var gr sourcev1.GitRepository - err := r.Get(context.Background(), dName, &gr) - if err != nil { - return sourcev1.GitRepositoryNotReady(repository, "DependencyNotReady", err.Error()), err - } - includedArtifacts = append(includedArtifacts, gr.GetArtifact()) + // Verify commit signature + if result, err := r.verifyCommitSignature(ctx, obj, *commit); err != nil || result.IsZero() { + return result, err } - // return early on unchanged revision and unchanged included repositories - if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && repository.GetArtifact().HasRevision(artifact.Revision) && !hasArtifactUpdated(repository.Status.IncludedArtifacts, includedArtifacts) { - if artifact.URL != repository.GetArtifact().URL { - r.Storage.SetArtifactURL(repository.GetArtifact()) - repository.Status.URL = r.Storage.SetHostname(repository.Status.URL) + // Create potential new artifact with current available metadata + *artifact = r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), commit.String(), fmt.Sprintf("%s.tar.gz", commit.Hash.String())) + + // Mark observations about the revision on the object + if !obj.GetArtifact().HasRevision(commit.String()) { + conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", "New upstream revision '%s'", commit.String()) + } + return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil +} + +// reconcileArtifact archives a new artifact to the storage, if the current observation on the object does not match the +// given data. +// +// The inspection of the given data to the object is differed, ensuring any stale observations as +// v1beta1.ArtifactUnavailableCondition and v1beta1.ArtifactOutdatedCondition are always deleted. +// If the given artifact and/or includes do not differ from the object's current, it returns early. +// Source ignore patterns are loaded, and the given directory is archived. +// On a successful archive, the artifact and includes in the status of the given object are set, and the symlink in the +// storage is updated to its path. +// +// The caller should assume a failure if an error is returned, or the Result is zero. +func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.GitRepository, artifact sourcev1.Artifact, includes artifactSet, dir string) (ctrl.Result, error) { + // Always restore the Ready condition in case it got removed due to a transient error + defer func() { + if obj.GetArtifact() != nil { + conditions.Delete(obj, sourcev1.ArtifactUnavailableCondition) } - return repository, nil + if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) { + conditions.Delete(obj, sourcev1.ArtifactOutdatedCondition) + conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, + "Stored artifact for revision '%s'", artifact.Revision) + } + }() + + // The artifact is up-to-date + if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) { + ctrl.LoggerFrom(ctx).Info("Artifact is up-to-date") + return ctrl.Result{RequeueAfter: obj.GetInterval().Duration}, nil } - // verify PGP signature - if repository.Spec.Verification != nil { - publicKeySecret := types.NamespacedName{ - Namespace: repository.Namespace, - Name: repository.Spec.Verification.SecretRef.Name, - } - secret := &corev1.Secret{} - if err := r.Client.Get(ctx, publicKeySecret, secret); err != nil { - err = fmt.Errorf("PGP public keys secret error: %w", err) - return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err - } - - var keyRings []string - for _, v := range secret.Data { - keyRings = append(keyRings, string(v)) - } - if _, err = commit.Verify(keyRings...); err != nil { - return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err - } + // Ensure target path exists and is a directory + if f, err := os.Stat(dir); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "failed to stat source path") + return ctrl.Result{}, err + } else if !f.IsDir() { + ctrl.LoggerFrom(ctx).Error(err, fmt.Sprintf("source path '%s' is not a directory", dir)) + return ctrl.Result{}, err } - // create artifact dir - err = r.Storage.MkdirAll(artifact) - if err != nil { - err = fmt.Errorf("mkdir dir error: %w", err) - return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + // Ensure artifact directory exists and acquire lock + if err := r.Storage.MkdirAll(artifact); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "failed to create artifact directory") + return ctrl.Result{}, err } - - for i, incl := range repository.Spec.Include { - toPath, err := securejoin.SecureJoin(tmpGit, incl.GetToPath()) - if err != nil { - return sourcev1.GitRepositoryNotReady(repository, "DependencyNotReady", err.Error()), err - } - err = r.Storage.CopyToPath(includedArtifacts[i], incl.GetFromPath(), toPath) - if err != nil { - return sourcev1.GitRepositoryNotReady(repository, "DependencyNotReady", err.Error()), err - } - } - - // acquire lock unlock, err := r.Storage.Lock(artifact) if err != nil { - err = fmt.Errorf("unable to acquire lock: %w", err) - return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + ctrl.LoggerFrom(ctx).Error(err, "failed to acquire lock for artifact") + return ctrl.Result{}, err } defer unlock() - // archive artifact and check integrity - ignoreDomain := strings.Split(tmpGit, string(filepath.Separator)) - ps, err := sourceignore.LoadIgnorePatterns(tmpGit, ignoreDomain) + // Load ignore rules for archiving + ps, err := sourceignore.LoadIgnorePatterns(dir, nil) if err != nil { - err = fmt.Errorf(".sourceignore error: %w", err) - return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + r.Eventf(obj, events.EventSeverityError, + "SourceIgnoreError", "Failed to load source ignore patterns from repository: %s", err) + return ctrl.Result{}, err } - if repository.Spec.Ignore != nil { - ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*repository.Spec.Ignore), ignoreDomain)...) - } - if err := r.Storage.Archive(&artifact, tmpGit, SourceIgnoreFilter(ps, ignoreDomain)); err != nil { - err = fmt.Errorf("storage archive error: %w", err) - return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + if obj.Spec.Ignore != nil { + ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*obj.Spec.Ignore), nil)...) } - // update latest symlink + // Archive directory to storage + if err := r.Storage.Archive(&artifact, dir, SourceIgnoreFilter(ps, nil)); err != nil { + r.Eventf(obj, events.EventSeverityError, sourcev1.StorageOperationFailedReason, + "Unable to archive artifact to storage: %s", err) + return ctrl.Result{}, err + } + r.AnnotatedEventf(obj, map[string]string{ + "revision": artifact.Revision, + "checksum": artifact.Checksum, + }, events.EventSeverityInfo, "NewArtifact", "Stored artifact for revision '%s'", artifact.Revision) + + // Record it on the object + obj.Status.Artifact = artifact.DeepCopy() + obj.Status.IncludedArtifacts = includes + + // Update symlink on a "best effort" basis url, err := r.Storage.Symlink(artifact, "latest.tar.gz") if err != nil { - err = fmt.Errorf("storage symlink error: %w", err) - return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + r.Eventf(obj, events.EventSeverityError, sourcev1.StorageOperationFailedReason, + "Failed to update status URL symlink: %s", err) } - - message := fmt.Sprintf("Fetched revision: %s", artifact.Revision) - return sourcev1.GitRepositoryReady(repository, artifact, includedArtifacts, url, sourcev1.GitOperationSucceedReason, message), nil + if url != "" { + obj.Status.URL = url + } + return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil } -func (r *GitRepositoryReconciler) reconcileDelete(ctx context.Context, repository sourcev1.GitRepository) (ctrl.Result, error) { - if err := r.gc(repository); err != nil { - r.event(ctx, repository, events.EventSeverityError, - fmt.Sprintf("garbage collection for deleted resource failed: %s", err.Error())) +// reconcileInclude reconciles the declared includes from the object by copying their artifact (sub)contents to the +// declared paths in the given directory. +// +// If an include is unavailable, it marks the object with v1beta1.IncludeUnavailableCondition and returns early. +// If the copy operations are successful, it deletes the v1beta1.IncludeUnavailableCondition from the object. +// If the artifactSet differs from the current set, it marks the object with v1beta1.ArtifactOutdatedCondition. +// +// The caller should assume a failure if an error is returned, or the Result is zero. +func (r *GitRepositoryReconciler) reconcileInclude(ctx context.Context, obj *sourcev1.GitRepository, artifacts artifactSet, dir string) (ctrl.Result, error) { + artifacts = make(artifactSet, len(obj.Spec.Include)) + for i, incl := range obj.Spec.Include { + // Do this first as it is much cheaper than copy operations + toPath, err := securejoin.SecureJoin(dir, incl.GetToPath()) + if err != nil { + conditions.MarkTrue(obj, sourcev1.IncludeUnavailableCondition, "IllegalPath", + "Path calculation for include %q failed: %s", incl.GitRepositoryRef.Name, err.Error()) + return ctrl.Result{}, err + } + + // Retrieve the included GitRepository + dep := &sourcev1.GitRepository{} + if err := r.Get(ctx, types.NamespacedName{Namespace: obj.Namespace, Name: incl.GitRepositoryRef.Name}, dep); err != nil { + conditions.MarkTrue(obj, sourcev1.IncludeUnavailableCondition, "NotFound", + "Could not get resource for include %q: %s", incl.GitRepositoryRef.Name, err.Error()) + return ctrl.Result{}, err + } + + // Confirm include has an artifact + if dep.GetArtifact() == nil { + conditions.MarkTrue(obj, sourcev1.IncludeUnavailableCondition, "NoArtifact", + "No artifact available for include %q", incl.GitRepositoryRef.Name) + return ctrl.Result{}, nil + } + + // Copy artifact (sub)contents to configured directory + if err := r.Storage.CopyToPath(dep.GetArtifact(), incl.GetFromPath(), toPath); err != nil { + conditions.MarkTrue(obj, sourcev1.IncludeUnavailableCondition, "CopyFailure", + "Failed to copy %q include from %s to %s: %s", incl.GitRepositoryRef.Name, incl.GetFromPath(), incl.GetToPath(), err.Error()) + r.Eventf(obj, events.EventSeverityError, sourcev1.IncludeUnavailableCondition, + "Failed to copy %q include from %s to %s: %s", incl.GitRepositoryRef.Name, incl.GetFromPath(), incl.GetToPath(), err.Error()) + return ctrl.Result{}, err + } + artifacts[i] = dep.GetArtifact().DeepCopy() + } + + // We now know all includes are available + conditions.Delete(obj, sourcev1.IncludeUnavailableCondition) + + // Observe if the artifacts still match the previous included ones + if artifacts.Diff(obj.Status.IncludedArtifacts) { + conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "IncludeChange", "Included artifacts differ from last observed includes") + } + return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil +} + +// reconcileDelete handles the delete of an object. It first garbage collects all artifacts for the object from the +// artifact storage, if successful, the finalizer is removed from the object. +func (r *GitRepositoryReconciler) reconcileDelete(ctx context.Context, obj *sourcev1.GitRepository) (ctrl.Result, error) { + // Garbage collect the resource's artifacts + if err := r.garbageCollect(ctx, obj); err != nil { // Return the error so we retry the failed garbage collection return ctrl.Result{}, err } - // Record deleted status - r.recordReadiness(ctx, repository) - - // Remove our finalizer from the list and update it - controllerutil.RemoveFinalizer(&repository, sourcev1.SourceFinalizer) - if err := r.Update(ctx, &repository); err != nil { - return ctrl.Result{}, err - } + // Remove our finalizer from the list + controllerutil.RemoveFinalizer(obj, sourcev1.SourceFinalizer) // Stop reconciliation as the object is being deleted return ctrl.Result{}, nil } -// resetStatus returns a modified v1beta1.GitRepository and a boolean indicating -// if the status field has been reset. -func (r *GitRepositoryReconciler) resetStatus(repository sourcev1.GitRepository) (sourcev1.GitRepository, bool) { - // We do not have an artifact, or it does no longer exist - if repository.GetArtifact() == nil || !r.Storage.ArtifactExist(*repository.GetArtifact()) { - repository = sourcev1.GitRepositoryProgressing(repository) - repository.Status.Artifact = nil - return repository, true +// verifyCommitSignature verifies the signature of the given commit if a verification mode is configured on the object. +func (r *GitRepositoryReconciler) verifyCommitSignature(ctx context.Context, obj *sourcev1.GitRepository, commit git.Commit) (ctrl.Result, error) { + // Check if there is a commit verification is configured and remove any old observations if there is none + if obj.Spec.Verification == nil || obj.Spec.Verification.Mode == "" { + conditions.Delete(obj, sourcev1.SourceVerifiedCondition) + return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil } - if repository.Generation != repository.Status.ObservedGeneration { - return sourcev1.GitRepositoryProgressing(repository), true + + // Get secret with GPG data + publicKeySecret := types.NamespacedName{ + Namespace: obj.Namespace, + Name: obj.Spec.Verification.SecretRef.Name, } - return repository, false + secret := &corev1.Secret{} + if err := r.Client.Get(ctx, publicKeySecret, secret); err != nil { + conditions.MarkFalse(obj, sourcev1.SourceVerifiedCondition, meta.FailedReason, "PGP public keys secret error: %s", err.Error()) + r.Eventf(obj, events.EventSeverityError, "VerificationError", "PGP public keys secret error: %s", err.Error()) + return ctrl.Result{}, err + } + + var keyRings []string + for _, v := range secret.Data { + keyRings = append(keyRings, string(v)) + } + // Verify commit with GPG data from secret + if _, err := commit.Verify(keyRings...); err != nil { + conditions.MarkFalse(obj, sourcev1.SourceVerifiedCondition, meta.FailedReason, "Signature verification of commit %q failed: %s", commit.Hash.String(), err) + r.Eventf(obj, events.EventSeverityError, "InvalidCommitSignature", "Signature verification of commit %q failed: %s", commit.Hash.String(), err) + // Return error in the hope the secret changes + return ctrl.Result{}, err + } + + conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, meta.SucceededReason, "Verified signature of commit %q", commit.Hash.String()) + r.Eventf(obj, events.EventSeverityInfo, "VerifiedCommit", "Verified signature of commit %q", commit.Hash.String()) + return ctrl.Result{RequeueAfter: obj.Spec.Interval.Duration}, nil } -// gc performs a garbage collection for the given v1beta1.GitRepository. -// It removes all but the current artifact except for when the -// deletion timestamp is set, which will result in the removal of -// all artifacts for the resource. -func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository) error { - if !repository.DeletionTimestamp.IsZero() { - return r.Storage.RemoveAll(r.Storage.NewArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "*")) +// garbageCollect performs a garbage collection for the given v1beta1.GitRepository. It removes all but the current +// artifact except for when the deletion timestamp is set, which will result in the removal of all artifacts for the +// resource. +func (r *GitRepositoryReconciler) garbageCollect(ctx context.Context, obj *sourcev1.GitRepository) error { + if !obj.DeletionTimestamp.IsZero() { + if err := r.Storage.RemoveAll(r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), "", "*")); err != nil { + r.Eventf(obj, events.EventSeverityError, "GarbageCollectionFailed", + "Garbage collection for deleted resource failed: %s", err) + return err + } + obj.Status.Artifact = nil + // TODO(hidde): we should only push this event if we actually garbage collected something + r.Eventf(obj, events.EventSeverityInfo, "GarbageCollectionSucceeded", + "Garbage collected artifacts for deleted resource") + return nil } - if repository.GetArtifact() != nil { - return r.Storage.RemoveAllButCurrent(*repository.GetArtifact()) + if obj.GetArtifact() != nil { + if err := r.Storage.RemoveAllButCurrent(*obj.GetArtifact()); err != nil { + r.Eventf(obj, events.EventSeverityError, "GarbageCollectionFailed", "Garbage collection of old artifacts failed: %s", err) + return err + } + // TODO(hidde): we should only push this event if we actually garbage collected something + r.Eventf(obj, events.EventSeverityInfo, "GarbageCollectionSucceeded", "Garbage collected old artifacts") } return nil } - -// event emits a Kubernetes event and forwards the event to notification controller if configured -func (r *GitRepositoryReconciler) event(ctx context.Context, repository sourcev1.GitRepository, severity, msg string) { - if r.EventRecorder != nil { - r.EventRecorder.Eventf(&repository, corev1.EventTypeNormal, severity, msg) - } - if r.ExternalEventRecorder != nil { - r.ExternalEventRecorder.Eventf(&repository, corev1.EventTypeNormal, severity, msg) - } -} - -func (r *GitRepositoryReconciler) recordReadiness(ctx context.Context, repository sourcev1.GitRepository) { - log := ctrl.LoggerFrom(ctx) - if r.MetricsRecorder == nil { - return - } - objRef, err := reference.GetReference(r.Scheme, &repository) - if err != nil { - log.Error(err, "unable to record readiness metric") - return - } - if rc := apimeta.FindStatusCondition(repository.Status.Conditions, meta.ReadyCondition); rc != nil { - r.MetricsRecorder.RecordCondition(*objRef, *rc, !repository.DeletionTimestamp.IsZero()) - } else { - r.MetricsRecorder.RecordCondition(*objRef, metav1.Condition{ - Type: meta.ReadyCondition, - Status: metav1.ConditionUnknown, - }, !repository.DeletionTimestamp.IsZero()) - } -} - -func (r *GitRepositoryReconciler) recordSuspension(ctx context.Context, gitrepository sourcev1.GitRepository) { - if r.MetricsRecorder == nil { - return - } - log := ctrl.LoggerFrom(ctx) - - objRef, err := reference.GetReference(r.Scheme, &gitrepository) - if err != nil { - log.Error(err, "unable to record suspended metric") - return - } - - if !gitrepository.DeletionTimestamp.IsZero() { - r.MetricsRecorder.RecordSuspend(*objRef, false) - } else { - r.MetricsRecorder.RecordSuspend(*objRef, gitrepository.Spec.Suspend) - } -} - -func (r *GitRepositoryReconciler) updateStatus(ctx context.Context, req ctrl.Request, newStatus sourcev1.GitRepositoryStatus) error { - var repository sourcev1.GitRepository - if err := r.Get(ctx, req.NamespacedName, &repository); err != nil { - return err - } - - patch := client.MergeFrom(repository.DeepCopy()) - repository.Status = newStatus - - return r.Status().Patch(ctx, &repository, patch) -} diff --git a/controllers/gitrepository_controller_test.go b/controllers/gitrepository_controller_test.go index 15910248..27f78a25 100644 --- a/controllers/gitrepository_controller_test.go +++ b/controllers/gitrepository_controller_test.go @@ -17,754 +17,1116 @@ limitations under the License. package controllers import ( - "context" - "crypto/tls" "fmt" - "net/http" "net/url" "os" - "os/exec" - "path" "path/filepath" "strings" + "testing" "time" "github.com/go-git/go-billy/v5/memfs" - "github.com/go-git/go-git/v5" + gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" - "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/transport/client" - httptransport "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/storage/memory" - . "github.com/onsi/ginkgo" - - . "github.com/onsi/ginkgo/extensions/table" + "github.com/go-logr/logr" . "github.com/onsi/gomega" + sshtestdata "golang.org/x/crypto/ssh/testdata" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/gittestserver" - "github.com/fluxcd/pkg/untar" + "github.com/fluxcd/pkg/runtime/conditions" + "github.com/fluxcd/pkg/ssh" + "github.com/fluxcd/pkg/testserver" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" + "github.com/fluxcd/source-controller/pkg/git" ) -var _ = Describe("GitRepositoryReconciler", func() { +var ( + testGitImplementations = []string{sourcev1.GoGitImplementation, sourcev1.LibGit2Implementation} +) - const ( - timeout = time.Second * 30 - interval = time.Second * 1 - indexInterval = time.Second * 1 - ) +func TestGitRepositoryReconciler_Reconcile(t *testing.T) { + g := NewWithT(t) - Context("GitRepository", func() { - var ( - namespace *corev1.Namespace - gitServer *gittestserver.GitServer - err error - ) + server, err := gittestserver.NewTempGitServer() + g.Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(server.Root()) + server.AutoCreate() + g.Expect(server.StartHTTP()).To(Succeed()) + defer server.StopHTTP() - BeforeEach(func() { - namespace = &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "git-repository-test" + randStringRunes(5)}, - } - err = k8sClient.Create(context.Background(), namespace) - Expect(err).NotTo(HaveOccurred(), "failed to create test namespace") + repoPath := "/test.git" + _, err = initGitRepo(server, "testdata/git/repository", git.DefaultBranch, repoPath) + g.Expect(err).NotTo(HaveOccurred()) - cert := corev1.Secret{ + obj := &sourcev1.GitRepository{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "gitrepository-reconcile-", + Namespace: "default", + }, + Spec: sourcev1.GitRepositorySpec{ + Interval: metav1.Duration{Duration: interval}, + URL: server.HTTPAddress() + repoPath, + }, + } + g.Expect(testEnv.Create(ctx, obj)).To(Succeed()) + + key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace} + + // Wait for finalizer to be set + g.Eventually(func() bool { + if err := testEnv.Get(ctx, key, obj); err != nil { + return false + } + return len(obj.Finalizers) > 0 + }, timeout).Should(BeTrue()) + + // Wait for GitRepository to be Ready + g.Eventually(func() bool { + if err := testEnv.Get(ctx, key, obj); err != nil { + return false + } + if !conditions.IsReady(obj) || obj.Status.Artifact == nil { + return false + } + readyCondition := conditions.Get(obj, meta.ReadyCondition) + return obj.Generation == readyCondition.ObservedGeneration && + obj.Generation == obj.Status.ObservedGeneration + }, timeout).Should(BeTrue()) + + g.Expect(testEnv.Delete(ctx, obj)).To(Succeed()) + + // Wait for GitRepository to be deleted + g.Eventually(func() bool { + if err := testEnv.Get(ctx, key, obj); err != nil { + return apierrors.IsNotFound(err) + } + return false + }, timeout).Should(BeTrue()) +} + +func TestGitRepositoryReconciler_reconcileSource_authStrategy(t *testing.T) { + type options struct { + username string + password string + publicKey []byte + privateKey []byte + ca []byte + } + + tests := []struct { + name string + skipForImplementation string + protocol string + server options + secret *corev1.Secret + beforeFunc func(obj *sourcev1.GitRepository) + want ctrl.Result + wantErr bool + assertConditions []metav1.Condition + }{ + { + name: "HTTP without secretRef makes ArtifactOutdated=True", + protocol: "http", + want: ctrl.Result{RequeueAfter: interval}, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "New upstream revision 'master/'"), + }, + }, + { + name: "HTTP with Basic Auth secret makes ArtifactOutdated=True", + protocol: "http", + server: options{ + username: "git", + password: "1234", + }, + secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: "cert", - Namespace: namespace.Name, + Name: "basic-auth", }, Data: map[string][]byte{ - "caFile": exampleCA, + "username": []byte("git"), + "password": []byte("1234"), }, - } - err = k8sClient.Create(context.Background(), &cert) - Expect(err).NotTo(HaveOccurred()) + }, + beforeFunc: func(obj *sourcev1.GitRepository) { + obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "basic-auth"} + }, + want: ctrl.Result{RequeueAfter: interval}, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "New upstream revision 'master/'"), + }, + }, + { + name: "HTTPS with CAFile secret makes ArtifactOutdated=True", + protocol: "https", + server: options{ + publicKey: tlsPublicKey, + privateKey: tlsPrivateKey, + ca: tlsCA, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ca-file", + }, + Data: map[string][]byte{ + "caFile": tlsCA, + }, + }, + beforeFunc: func(obj *sourcev1.GitRepository) { + obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "ca-file"} + }, + want: ctrl.Result{RequeueAfter: interval}, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "New upstream revision 'master/'"), + }, + }, + { + name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error", + skipForImplementation: sourcev1.LibGit2Implementation, + protocol: "https", + server: options{ + publicKey: tlsPublicKey, + privateKey: tlsPrivateKey, + ca: tlsCA, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-ca", + }, + Data: map[string][]byte{ + "caFile": []byte("invalid"), + }, + }, + beforeFunc: func(obj *sourcev1.GitRepository) { + obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "invalid-ca"} + }, + wantErr: true, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.CheckoutFailedCondition, sourcev1.GitOperationFailedReason, "x509: certificate signed by unknown authority"), + }, + }, + { + name: "HTTPS with invalid CAFile secret makes CheckoutFailed=True and returns error", + skipForImplementation: sourcev1.GoGitImplementation, + protocol: "https", + server: options{ + publicKey: tlsPublicKey, + privateKey: tlsPrivateKey, + ca: tlsCA, + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "invalid-ca", + }, + Data: map[string][]byte{ + "caFile": []byte("invalid"), + }, + }, + beforeFunc: func(obj *sourcev1.GitRepository) { + obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "invalid-ca"} + }, + wantErr: true, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.CheckoutFailedCondition, sourcev1.GitOperationFailedReason, "Failed to checkout and determine revision: unable to clone '', error: Certificate"), + }, + }, + { + name: "SSH with private key secret makes ArtifactOutdated=True", + protocol: "ssh", + server: options{ + username: "git", + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "private-key", + }, + Data: map[string][]byte{ + "username": []byte("git"), + "identity": sshtestdata.PEMBytes["rsa"], + }, + }, + beforeFunc: func(obj *sourcev1.GitRepository) { + obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "private-key"} + }, + want: ctrl.Result{RequeueAfter: interval}, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "New upstream revision 'master/'"), + }, + }, + { + name: "SSH with password protected private key secret makes ArtifactOutdated=True", + protocol: "ssh", + server: options{ + username: "git", + }, + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "private-key", + }, + Data: map[string][]byte{ + "username": []byte("git"), + "identity": sshtestdata.PEMEncryptedKeys[2].PEMBytes, + "password": []byte("password"), + }, + }, + beforeFunc: func(obj *sourcev1.GitRepository) { + obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "private-key"} + }, + want: ctrl.Result{RequeueAfter: interval}, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "New upstream revision 'master/'"), + }, + }, + { + name: "Include get failure makes CheckoutFailed=True and returns error", + protocol: "http", + server: options{ + username: "git", + }, + beforeFunc: func(obj *sourcev1.GitRepository) { + obj.Spec.SecretRef = &meta.LocalObjectReference{Name: "non-existing"} + }, + wantErr: true, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.CheckoutFailedCondition, "AuthenticationFailed", "Failed to get secret '/non-existing': secrets \"non-existing\" not found"), + }, + }, + } - gitServer, err = gittestserver.NewTempGitServer() - Expect(err).NotTo(HaveOccurred()) - gitServer.AutoCreate() - }) - - AfterEach(func() { - os.RemoveAll(gitServer.Root()) - - err = k8sClient.Delete(context.Background(), namespace) - Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace") - }) - - type refTestCase struct { - reference *sourcev1.GitRepositoryRef - createRefs []string - - waitForReason string - - expectStatus metav1.ConditionStatus - expectMessage string - expectRevision string - - secretRef *meta.LocalObjectReference - gitImplementation string + for _, tt := range tests { + obj := &sourcev1.GitRepository{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "auth-strategy-", + }, + Spec: sourcev1.GitRepositorySpec{ + Interval: metav1.Duration{Duration: interval}, + Timeout: &metav1.Duration{Duration: interval}, + }, } - DescribeTable("Git references tests", func(t refTestCase) { - err = gitServer.StartHTTP() - defer gitServer.StopHTTP() - Expect(err).NotTo(HaveOccurred()) + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) - u, err := url.Parse(gitServer.HTTPAddress()) - Expect(err).NotTo(HaveOccurred()) - u.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5))) + server, err := gittestserver.NewTempGitServer() + g.Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(server.Root()) + server.AutoCreate() - fs := memfs.New() - gitrepo, err := git.Init(memory.NewStorage(), fs) - Expect(err).NotTo(HaveOccurred()) + repoPath := "/test.git" + localRepo, err := initGitRepo(server, "testdata/git/repository", git.DefaultBranch, repoPath) + g.Expect(err).NotTo(HaveOccurred()) - wt, err := gitrepo.Worktree() - Expect(err).NotTo(HaveOccurred()) - - ff, _ := fs.Create("fixture") - _ = ff.Close() - _, err = wt.Add(fs.Join("fixture")) - Expect(err).NotTo(HaveOccurred()) - - commit, err := wt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{ - Name: "John Doe", - Email: "john@example.com", - When: time.Now(), - }}) - Expect(err).NotTo(HaveOccurred()) - - for _, ref := range t.createRefs { - hRef := plumbing.NewHashReference(plumbing.ReferenceName(ref), commit) - err = gitrepo.Storer.SetReference(hRef) - Expect(err).NotTo(HaveOccurred()) + if len(tt.server.username+tt.server.password) > 0 { + server.Auth(tt.server.username, tt.server.password) } - remote, err := gitrepo.CreateRemote(&config.RemoteConfig{ - Name: "origin", - URLs: []string{u.String()}, - }) - Expect(err).NotTo(HaveOccurred()) + secret := tt.secret.DeepCopy() + switch tt.protocol { + case "http": + g.Expect(server.StartHTTP()).To(Succeed()) + defer server.StopHTTP() + obj.Spec.URL = server.HTTPAddress() + repoPath + case "https": + g.Expect(server.StartHTTPS(tt.server.publicKey, tt.server.privateKey, tt.server.ca, "example.com")).To(Succeed()) + obj.Spec.URL = server.HTTPAddress() + repoPath + case "ssh": + server.KeyDir(filepath.Join(server.Root(), "keys")) - err = remote.Push(&git.PushOptions{ - RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}, - }) - Expect(err).NotTo(HaveOccurred()) + g.Expect(server.ListenSSH()).To(Succeed()) + obj.Spec.URL = server.SSHAddress() + repoPath - t.reference.Commit = strings.Replace(t.reference.Commit, "", commit.String(), 1) + go func() { + server.StartSSH() + }() + defer server.StopSSH() - key := types.NamespacedName{ - Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)), - Namespace: namespace.Name, + if secret != nil && len(secret.Data["known_hosts"]) == 0 { + u, err := url.Parse(obj.Spec.URL) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(u.Host).ToNot(BeEmpty()) + knownHosts, err := ssh.ScanHostKey(u.Host, timeout) + g.Expect(err).NotTo(HaveOccurred()) + secret.Data["known_hosts"] = knownHosts + } + default: + t.Fatalf("unsupported protocol %q", tt.protocol) } - created := &sourcev1.GitRepository{ + + if tt.beforeFunc != nil { + tt.beforeFunc(obj) + } + + builder := fakeclient.NewClientBuilder().WithScheme(testEnv.GetScheme()) + if secret != nil { + builder.WithObjects(secret.DeepCopy()) + } + + r := &GitRepositoryReconciler{ + Client: builder.Build(), + Storage: testStorage, + } + + for _, i := range testGitImplementations { + t.Run(i, func(t *testing.T) { + g := NewWithT(t) + + if tt.skipForImplementation == i { + t.Skipf("Skipped for Git implementation %q", i) + } + + tmpDir, err := os.MkdirTemp("", "auth-strategy-") + g.Expect(err).To(BeNil()) + defer os.RemoveAll(tmpDir) + + obj := obj.DeepCopy() + obj.Spec.GitImplementation = i + + head, _ := localRepo.Head() + assertConditions := tt.assertConditions + for k := range assertConditions { + assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", head.Hash().String()) + assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", obj.Spec.URL) + } + + var artifact sourcev1.Artifact + dlog := log.NewDelegatingLogSink(log.NullLogSink{}) + nullLogger := logr.New(dlog) + got, err := r.reconcileSource(logr.NewContext(ctx, nullLogger), obj, &artifact, tmpDir) + g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions)) + g.Expect(err != nil).To(Equal(tt.wantErr)) + g.Expect(got).To(Equal(tt.want)) + g.Expect(artifact).ToNot(BeNil()) + }) + } + }) + } +} + +func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T) { + g := NewWithT(t) + + branches := []string{"staging"} + tags := []string{"non-semver-tag", "v0.1.0", "0.2.0", "v0.2.1", "v1.0.0-alpha", "v1.1.0", "v2.0.0"} + + tests := []struct { + name string + reference *sourcev1.GitRepositoryRef + want ctrl.Result + wantErr bool + wantRevision string + }{ + { + name: "Nil reference (default branch)", + want: ctrl.Result{RequeueAfter: interval}, + wantRevision: "master/", + }, + { + name: "Branch", + reference: &sourcev1.GitRepositoryRef{ + Branch: "staging", + }, + want: ctrl.Result{RequeueAfter: interval}, + wantRevision: "staging/", + }, + { + name: "Tag", + reference: &sourcev1.GitRepositoryRef{ + Tag: "v0.1.0", + }, + want: ctrl.Result{RequeueAfter: interval}, + wantRevision: "v0.1.0/", + }, + { + name: "Branch commit", + reference: &sourcev1.GitRepositoryRef{ + Branch: "staging", + Commit: "", + }, + want: ctrl.Result{RequeueAfter: interval}, + wantRevision: "staging/", + }, + { + name: "SemVer", + reference: &sourcev1.GitRepositoryRef{ + SemVer: "*", + }, + want: ctrl.Result{RequeueAfter: interval}, + wantRevision: "v2.0.0/", + }, + { + name: "SemVer range", + reference: &sourcev1.GitRepositoryRef{ + SemVer: "", + }, + { + name: "SemVer prerelease", + reference: &sourcev1.GitRepositoryRef{ + SemVer: ">=1.0.0-0 <1.1.0-0", + }, + wantRevision: "v1.0.0-alpha/", + want: ctrl.Result{RequeueAfter: interval}, + }, + } + + server, err := gittestserver.NewTempGitServer() + g.Expect(err).To(BeNil()) + server.AutoCreate() + g.Expect(server.StartHTTP()).To(Succeed()) + defer server.StopHTTP() + + repoPath := "/test.git" + localRepo, err := initGitRepo(server, "testdata/git/repository", git.DefaultBranch, repoPath) + g.Expect(err).NotTo(HaveOccurred()) + + headRef, err := localRepo.Head() + g.Expect(err).NotTo(HaveOccurred()) + + for _, branch := range branches { + g.Expect(remoteBranchForHead(localRepo, headRef, branch)).To(Succeed()) + } + for _, tag := range tags { + g.Expect(remoteTagForHead(localRepo, headRef, tag)).To(Succeed()) + } + + r := &GitRepositoryReconciler{ + Client: fakeclient.NewClientBuilder().WithScheme(runtime.NewScheme()).Build(), + Storage: testStorage, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + obj := &sourcev1.GitRepository{ ObjectMeta: metav1.ObjectMeta{ - Name: key.Name, - Namespace: key.Namespace, + GenerateName: "checkout-strategy-", }, Spec: sourcev1.GitRepositorySpec{ - URL: u.String(), - Interval: metav1.Duration{Duration: indexInterval}, - Reference: t.reference, + Interval: metav1.Duration{Duration: interval}, + Timeout: &metav1.Duration{Duration: interval}, + URL: server.HTTPAddress() + repoPath, + Reference: tt.reference, }, } - Expect(k8sClient.Create(context.Background(), created)).Should(Succeed()) - defer k8sClient.Delete(context.Background(), created) - got := &sourcev1.GitRepository{} - var cond metav1.Condition - Eventually(func() bool { - _ = k8sClient.Get(context.Background(), key, got) - for _, c := range got.Status.Conditions { - if c.Reason == t.waitForReason { - cond = c - return true + if obj.Spec.Reference != nil && obj.Spec.Reference.Commit == "" { + obj.Spec.Reference.Commit = headRef.Hash().String() + } + + for _, i := range testGitImplementations { + t.Run(i, func(t *testing.T) { + g := NewWithT(t) + + tmpDir, err := os.MkdirTemp("", "checkout-strategy-") + g.Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + obj := obj.DeepCopy() + obj.Spec.GitImplementation = i + + var artifact sourcev1.Artifact + got, err := r.reconcileSource(ctx, obj, &artifact, tmpDir) + if err != nil { + println(err.Error()) } - } - return false - }, timeout, interval).Should(BeTrue()) - - Expect(cond.Status).To(Equal(t.expectStatus)) - Expect(cond.Message).To(ContainSubstring(t.expectMessage)) - Expect(got.Status.Artifact == nil).To(Equal(t.expectRevision == "")) - if t.expectRevision != "" { - Expect(got.Status.Artifact.Revision).To(Equal(t.expectRevision + "/" + commit.String())) + g.Expect(err != nil).To(Equal(tt.wantErr)) + g.Expect(got).To(Equal(tt.want)) + if tt.wantRevision != "" { + revision := strings.ReplaceAll(tt.wantRevision, "", headRef.Hash().String()) + g.Expect(artifact.Revision).To(Equal(revision)) + g.Expect(conditions.IsTrue(obj, sourcev1.ArtifactOutdatedCondition)).To(BeTrue()) + } + }) } + }) + } +} + +func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) { + tests := []struct { + name string + dir string + beforeFunc func(obj *sourcev1.GitRepository) + afterFunc func(t *WithT, obj *sourcev1.GitRepository, artifact sourcev1.Artifact) + want ctrl.Result + wantErr bool + assertConditions []metav1.Condition + }{ + { + name: "Archiving artifact to storage makes Ready=True", + dir: "testdata/git/repository", + beforeFunc: func(obj *sourcev1.GitRepository) { + obj.Spec.Interval = metav1.Duration{Duration: interval} + }, + afterFunc: func(t *WithT, obj *sourcev1.GitRepository, artifact sourcev1.Artifact) { + t.Expect(obj.GetArtifact()).ToNot(BeNil()) + t.Expect(obj.GetArtifact().Checksum).To(Equal("f9955588f6aeed7be9b1ef15cd2ddac47bb53291")) + }, + want: ctrl.Result{RequeueAfter: interval}, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(meta.ReadyCondition, meta.SucceededReason, "Stored artifact for revision 'main/revision'"), + }, }, - Entry("branch", refTestCase{ - reference: &sourcev1.GitRepositoryRef{Branch: "some-branch"}, - createRefs: []string{"refs/heads/some-branch"}, - waitForReason: sourcev1.GitOperationSucceedReason, - expectStatus: metav1.ConditionTrue, - expectRevision: "some-branch", - }), - Entry("branch non existing", refTestCase{ - reference: &sourcev1.GitRepositoryRef{Branch: "invalid-branch"}, - waitForReason: sourcev1.GitOperationFailedReason, - expectStatus: metav1.ConditionFalse, - expectMessage: "couldn't find remote ref", - }), - Entry("tag", refTestCase{ - reference: &sourcev1.GitRepositoryRef{Tag: "some-tag"}, - createRefs: []string{"refs/tags/some-tag"}, - waitForReason: sourcev1.GitOperationSucceedReason, - expectStatus: metav1.ConditionTrue, - expectRevision: "some-tag", - }), - Entry("tag non existing", refTestCase{ - reference: &sourcev1.GitRepositoryRef{Tag: "invalid-tag"}, - waitForReason: sourcev1.GitOperationFailedReason, - expectStatus: metav1.ConditionFalse, - expectMessage: "couldn't find remote ref", - }), - Entry("semver", refTestCase{ - reference: &sourcev1.GitRepositoryRef{SemVer: "1.0.0"}, - createRefs: []string{"refs/tags/v1.0.0"}, - waitForReason: sourcev1.GitOperationSucceedReason, - expectStatus: metav1.ConditionTrue, - expectRevision: "v1.0.0", - }), - Entry("semver range", refTestCase{ - reference: &sourcev1.GitRepositoryRef{SemVer: ">=0.1.0 <1.0.0"}, - createRefs: []string{"refs/tags/0.1.0", "refs/tags/0.1.1", "refs/tags/0.2.0", "refs/tags/1.0.0"}, - waitForReason: sourcev1.GitOperationSucceedReason, - expectStatus: metav1.ConditionTrue, - expectRevision: "0.2.0", - }), - Entry("mixed semver range", refTestCase{ - reference: &sourcev1.GitRepositoryRef{SemVer: ">=0.1.0 <1.0.0"}, - createRefs: []string{"refs/tags/0.1.0", "refs/tags/v0.1.1", "refs/tags/v0.2.0", "refs/tags/1.0.0"}, - waitForReason: sourcev1.GitOperationSucceedReason, - expectStatus: metav1.ConditionTrue, - expectRevision: "v0.2.0", - }), - Entry("semver invalid", refTestCase{ - reference: &sourcev1.GitRepositoryRef{SemVer: "1.2.3.4"}, - waitForReason: sourcev1.GitOperationFailedReason, - expectStatus: metav1.ConditionFalse, - expectMessage: "semver parse error: improper constraint: 1.2.3.4", - }), - Entry("semver no match", refTestCase{ - reference: &sourcev1.GitRepositoryRef{SemVer: "1.0.0"}, - waitForReason: sourcev1.GitOperationFailedReason, - expectStatus: metav1.ConditionFalse, - expectMessage: "no match found for semver: 1.0.0", - }), - Entry("commit", refTestCase{ - reference: &sourcev1.GitRepositoryRef{ - Commit: "", - }, - waitForReason: sourcev1.GitOperationSucceedReason, - expectStatus: metav1.ConditionTrue, - expectRevision: "HEAD", - }), - Entry("commit in branch", refTestCase{ - reference: &sourcev1.GitRepositoryRef{ - Branch: "some-branch", - Commit: "", - }, - createRefs: []string{"refs/heads/some-branch"}, - waitForReason: sourcev1.GitOperationSucceedReason, - expectStatus: metav1.ConditionTrue, - expectRevision: "some-branch", - }), - Entry("invalid commit", refTestCase{ - reference: &sourcev1.GitRepositoryRef{ - Branch: "master", - Commit: "invalid", - }, - waitForReason: sourcev1.GitOperationFailedReason, - expectStatus: metav1.ConditionFalse, - expectMessage: "failed to resolve commit object for 'invalid': object not found", - }), - ) + { + name: "Spec ignore overwrite is taken into account", + dir: "testdata/git/repository", + beforeFunc: func(obj *sourcev1.GitRepository) { + obj.Spec.Interval = metav1.Duration{Duration: interval} + obj.Spec.Ignore = pointer.StringPtr("!**.txt\n") + }, + afterFunc: func(t *WithT, obj *sourcev1.GitRepository, artifact sourcev1.Artifact) { + t.Expect(obj.GetArtifact()).ToNot(BeNil()) + t.Expect(obj.GetArtifact().Checksum).To(Equal("542a8ad0171118a3249e8c531c598b898defd742")) + }, + want: ctrl.Result{RequeueAfter: interval}, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(meta.ReadyCondition, meta.SucceededReason, "Stored artifact for revision 'main/revision'"), + }, + }, + } - DescribeTable("Git self signed cert tests", func(t refTestCase) { - err = gitServer.StartHTTPS(examplePublicKey, examplePrivateKey, exampleCA, "example.com") - defer gitServer.StopHTTP() - Expect(err).NotTo(HaveOccurred()) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) - u, err := url.Parse(gitServer.HTTPAddress()) - Expect(err).NotTo(HaveOccurred()) - u.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5))) - - var transport = httptransport.NewClient(&http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - }) - client.InstallProtocol("https", transport) - - fs := memfs.New() - gitrepo, err := git.Init(memory.NewStorage(), fs) - Expect(err).NotTo(HaveOccurred()) - - wt, err := gitrepo.Worktree() - Expect(err).NotTo(HaveOccurred()) - - ff, _ := fs.Create("fixture") - _ = ff.Close() - _, err = wt.Add(fs.Join("fixture")) - Expect(err).NotTo(HaveOccurred()) - - commit, err := wt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{ - Name: "John Doe", - Email: "john@example.com", - When: time.Now(), - }}) - Expect(err).NotTo(HaveOccurred()) - - for _, ref := range t.createRefs { - hRef := plumbing.NewHashReference(plumbing.ReferenceName(ref), commit) - err = gitrepo.Storer.SetReference(hRef) - Expect(err).NotTo(HaveOccurred()) + r := &GitRepositoryReconciler{ + Storage: testStorage, } - remote, err := gitrepo.CreateRemote(&config.RemoteConfig{ - Name: "origin", - URLs: []string{u.String()}, - }) - Expect(err).NotTo(HaveOccurred()) - - err = remote.Push(&git.PushOptions{ - RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}, - }) - Expect(err).NotTo(HaveOccurred()) - - t.reference.Commit = strings.Replace(t.reference.Commit, "", commit.String(), 1) - - client.InstallProtocol("https", httptransport.DefaultClient) - - key := types.NamespacedName{ - Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)), - Namespace: namespace.Name, - } - created := &sourcev1.GitRepository{ + obj := &sourcev1.GitRepository{ ObjectMeta: metav1.ObjectMeta{ - Name: key.Name, - Namespace: key.Namespace, - }, - Spec: sourcev1.GitRepositorySpec{ - URL: u.String(), - Interval: metav1.Duration{Duration: indexInterval}, - Reference: t.reference, - GitImplementation: t.gitImplementation, - SecretRef: t.secretRef, + GenerateName: "reconcile-artifact-", + Generation: 1, }, + Status: sourcev1.GitRepositoryStatus{}, } - Expect(k8sClient.Create(context.Background(), created)).Should(Succeed()) - defer k8sClient.Delete(context.Background(), created) - got := &sourcev1.GitRepository{} - var cond metav1.Condition - Eventually(func() bool { - _ = k8sClient.Get(context.Background(), key, got) - for _, c := range got.Status.Conditions { - if c.Reason == t.waitForReason { - cond = c - return true - } - } - return false - }, timeout, interval).Should(BeTrue()) + if tt.beforeFunc != nil { + tt.beforeFunc(obj) + } - Expect(cond.Status).To(Equal(t.expectStatus)) - Expect(cond.Message).To(ContainSubstring(t.expectMessage)) - Expect(got.Status.Artifact == nil).To(Equal(t.expectRevision == "")) + artifact := testStorage.NewArtifactFor(obj.Kind, obj, "main/revision", "checksum.tar.gz") + + got, err := r.reconcileArtifact(ctx, obj, artifact, nil, tt.dir) + g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions)) + g.Expect(err != nil).To(Equal(tt.wantErr)) + g.Expect(got).To(Equal(tt.want)) + + if tt.afterFunc != nil { + tt.afterFunc(g, obj, artifact) + } + }) + } +} + +func TestGitRepositoryReconciler_reconcileInclude(t *testing.T) { + g := NewWithT(t) + + server, err := testserver.NewTempArtifactServer() + g.Expect(err).NotTo(HaveOccurred()) + storage, err := newTestStorage(server.HTTPServer) + g.Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(testStorage.BasePath) + + dependencyInterval := 5 * time.Second + + type dependency struct { + name string + withArtifact bool + conditions []metav1.Condition + } + + type include struct { + name string + fromPath string + toPath string + shouldExist bool + } + + tests := []struct { + name string + dependencies []dependency + includes []include + beforeFunc func(obj *sourcev1.GitRepository) + want ctrl.Result + wantErr bool + assertConditions []metav1.Condition + }{ + { + name: "New includes make ArtifactOutdated=True", + dependencies: []dependency{ + { + name: "a", + withArtifact: true, + conditions: []metav1.Condition{ + *conditions.TrueCondition(meta.ReadyCondition, "Foo", "foo ready"), + }, + }, + { + name: "b", + withArtifact: true, + conditions: []metav1.Condition{ + *conditions.TrueCondition(meta.ReadyCondition, "Bar", "bar ready"), + }, + }, + }, + includes: []include{ + {name: "a", toPath: "a/"}, + {name: "b", toPath: "b/"}, + }, + want: ctrl.Result{RequeueAfter: interval}, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "IncludeChange", "Included artifacts differ from last observed includes"), + }, }, - Entry("self signed libgit2 without CA", refTestCase{ - reference: &sourcev1.GitRepositoryRef{Branch: "main"}, - waitForReason: sourcev1.GitOperationFailedReason, - expectStatus: metav1.ConditionFalse, - expectMessage: "user rejected certificate", - gitImplementation: sourcev1.LibGit2Implementation, - }), - Entry("self signed libgit2 with CA", refTestCase{ - reference: &sourcev1.GitRepositoryRef{Branch: "some-branch"}, - createRefs: []string{"refs/heads/some-branch"}, - waitForReason: sourcev1.GitOperationSucceedReason, - expectStatus: metav1.ConditionTrue, - expectRevision: "some-branch", - secretRef: &meta.LocalObjectReference{Name: "cert"}, - gitImplementation: sourcev1.LibGit2Implementation, - }), - Entry("self signed go-git without CA", refTestCase{ - reference: &sourcev1.GitRepositoryRef{Branch: "main"}, - waitForReason: sourcev1.GitOperationFailedReason, - expectStatus: metav1.ConditionFalse, - expectMessage: "x509: certificate signed by unknown authority", - }), - Entry("self signed go-git with CA", refTestCase{ - reference: &sourcev1.GitRepositoryRef{Branch: "some-branch"}, - createRefs: []string{"refs/heads/some-branch"}, - waitForReason: sourcev1.GitOperationSucceedReason, - expectStatus: metav1.ConditionTrue, - expectRevision: "some-branch", - secretRef: &meta.LocalObjectReference{Name: "cert"}, - gitImplementation: sourcev1.GoGitImplementation, - }), - ) + { + name: "Include get failure makes IncludeUnavailable=True and returns error", + includes: []include{ + {name: "a", toPath: "a/"}, + }, + wantErr: true, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.IncludeUnavailableCondition, "NotFound", "Could not get resource for include \"a\": gitrepositories.source.toolkit.fluxcd.io \"a\" not found"), + }, + }, + { + name: "Include without an artifact makes IncludeUnavailable=True", + dependencies: []dependency{ + { + name: "a", + withArtifact: false, + conditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.IncludeUnavailableCondition, "Foo", "foo unavailable"), + }, + }, + }, + includes: []include{ + {name: "a", toPath: "a/"}, + }, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.IncludeUnavailableCondition, "NoArtifact", "No artifact available for include \"a\""), + }, + }, + { + name: "Invalid FromPath makes IncludeUnavailable=True and returns error", + dependencies: []dependency{ + { + name: "a", + withArtifact: true, + }, + }, + includes: []include{ + {name: "a", fromPath: "../../../path", shouldExist: false}, + }, + wantErr: true, + assertConditions: []metav1.Condition{ + *conditions.TrueCondition(sourcev1.IncludeUnavailableCondition, "CopyFailure", "unpack/path: no such file or directory"), + }, + }, + { + name: "Outdated IncludeUnavailable is removed", + beforeFunc: func(obj *sourcev1.GitRepository) { + conditions.MarkTrue(obj, sourcev1.IncludeUnavailableCondition, "NoArtifact", "") + }, + want: ctrl.Result{RequeueAfter: interval}, + assertConditions: []metav1.Condition{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) - Context("recurse submodules", func() { - It("downloads submodules when asked", func() { - Expect(gitServer.StartHTTP()).To(Succeed()) - defer gitServer.StopHTTP() - - u, err := url.Parse(gitServer.HTTPAddress()) - Expect(err).NotTo(HaveOccurred()) - - subRepoURL := *u - subRepoURL.Path = path.Join(u.Path, fmt.Sprintf("subrepository-%s.git", randStringRunes(5))) - - // create the git repo to use as a submodule - fs := memfs.New() - subRepo, err := git.Init(memory.NewStorage(), fs) - Expect(err).NotTo(HaveOccurred()) - - wt, err := subRepo.Worktree() - Expect(err).NotTo(HaveOccurred()) - - ff, _ := fs.Create("fixture") - _ = ff.Close() - _, err = wt.Add(fs.Join("fixture")) - Expect(err).NotTo(HaveOccurred()) - - _, err = wt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{ - Name: "John Doe", - Email: "john@example.com", - When: time.Now(), - }}) - Expect(err).NotTo(HaveOccurred()) - - remote, err := subRepo.CreateRemote(&config.RemoteConfig{ - Name: "origin", - URLs: []string{subRepoURL.String()}, - }) - Expect(err).NotTo(HaveOccurred()) - - err = remote.Push(&git.PushOptions{ - RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}, - }) - Expect(err).NotTo(HaveOccurred()) - - // this one is linked to a real directory, so that I can - // exec `git submodule add` later - tmp, err := os.MkdirTemp("", "flux-test") - Expect(err).NotTo(HaveOccurred()) - defer os.RemoveAll(tmp) - - repoDir := filepath.Join(tmp, "git") - repo, err := git.PlainInit(repoDir, false) - Expect(err).NotTo(HaveOccurred()) - - wt, err = repo.Worktree() - Expect(err).NotTo(HaveOccurred()) - _, err = wt.Commit("Initial revision", &git.CommitOptions{ - Author: &object.Signature{ - Name: "John Doe", - Email: "john@example.com", - When: time.Now(), - }}) - Expect(err).NotTo(HaveOccurred()) - - submodAdd := exec.Command("git", "submodule", "add", "-b", "master", subRepoURL.String(), "sub") - submodAdd.Dir = repoDir - out, err := submodAdd.CombinedOutput() - os.Stdout.Write(out) - Expect(err).NotTo(HaveOccurred()) - - _, err = wt.Commit("Add submodule", &git.CommitOptions{ - Author: &object.Signature{ - Name: "John Doe", - Email: "john@example.com", - When: time.Now(), - }}) - Expect(err).NotTo(HaveOccurred()) - - mainRepoURL := *u - mainRepoURL.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5))) - remote, err = repo.CreateRemote(&config.RemoteConfig{ - Name: "origin", - URLs: []string{mainRepoURL.String()}, - }) - Expect(err).NotTo(HaveOccurred()) - - err = remote.Push(&git.PushOptions{ - RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}, - }) - Expect(err).NotTo(HaveOccurred()) - - key := types.NamespacedName{ - Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)), - Namespace: namespace.Name, - } - created := &sourcev1.GitRepository{ + var depObjs []client.Object + for _, d := range tt.dependencies { + obj := &sourcev1.GitRepository{ ObjectMeta: metav1.ObjectMeta{ - Name: key.Name, - Namespace: key.Namespace, + Name: d.name, }, - Spec: sourcev1.GitRepositorySpec{ - URL: mainRepoURL.String(), - Interval: metav1.Duration{Duration: indexInterval}, - Reference: &sourcev1.GitRepositoryRef{Branch: "master"}, - GitImplementation: sourcev1.GoGitImplementation, // only works with go-git - RecurseSubmodules: true, + Status: sourcev1.GitRepositoryStatus{ + Conditions: d.conditions, }, } - Expect(k8sClient.Create(context.Background(), created)).Should(Succeed()) - defer k8sClient.Delete(context.Background(), created) - - got := &sourcev1.GitRepository{} - Eventually(func() bool { - _ = k8sClient.Get(context.Background(), key, got) - for _, c := range got.Status.Conditions { - if c.Reason == sourcev1.GitOperationSucceedReason { - return true - } + if d.withArtifact { + obj.Status.Artifact = &sourcev1.Artifact{ + Path: d.name + ".tar.gz", + Revision: d.name, + LastUpdateTime: metav1.Now(), } - return false - }, timeout, interval).Should(BeTrue()) + g.Expect(storage.Archive(obj.GetArtifact(), "testdata/git/repository", nil)).To(Succeed()) + } + depObjs = append(depObjs, obj) + } - // check that the downloaded artifact includes the - // file from the submodule - res, err := http.Get(got.Status.URL) - Expect(err).NotTo(HaveOccurred()) - Expect(res.StatusCode).To(Equal(http.StatusOK)) + builder := fakeclient.NewClientBuilder().WithScheme(testEnv.GetScheme()) + if len(tt.dependencies) > 0 { + builder.WithObjects(depObjs...) + } - _, err = untar.Untar(res.Body, filepath.Join(tmp, "tar")) - Expect(err).NotTo(HaveOccurred()) - Expect(filepath.Join(tmp, "tar", "sub", "fixture")).To(BeAnExistingFile()) - }) + r := &GitRepositoryReconciler{ + Client: builder.Build(), + EventRecorder: record.NewFakeRecorder(32), + // Events: helper.Events{ + // Scheme: testEnv.GetScheme(), + // EventRecorder: record.NewFakeRecorder(32), + // }, + Storage: storage, + requeueDependency: dependencyInterval, + } + + obj := &sourcev1.GitRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "reconcile-include", + }, + Spec: sourcev1.GitRepositorySpec{ + Interval: metav1.Duration{Duration: interval}, + }, + } + + for i, incl := range tt.includes { + incl := sourcev1.GitRepositoryInclude{ + GitRepositoryRef: meta.LocalObjectReference{Name: incl.name}, + FromPath: incl.fromPath, + ToPath: incl.toPath, + } + tt.includes[i].fromPath = incl.GetFromPath() + tt.includes[i].toPath = incl.GetToPath() + obj.Spec.Include = append(obj.Spec.Include, incl) + } + + if tt.beforeFunc != nil { + tt.beforeFunc(obj) + } + + tmpDir, err := os.MkdirTemp("", "include-") + g.Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + var artifacts artifactSet + got, err := r.reconcileInclude(ctx, obj, artifacts, tmpDir) + g.Expect(obj.GetConditions()).To(conditions.MatchConditions(tt.assertConditions)) + g.Expect(err != nil).To(Equal(tt.wantErr)) + g.Expect(got).To(Equal(tt.want)) + for _, i := range tt.includes { + if i.toPath != "" { + expect := g.Expect(filepath.Join(testStorage.BasePath, i.toPath)) + if i.shouldExist { + expect.To(BeADirectory()) + } else { + expect.NotTo(BeADirectory()) + } + } + if i.shouldExist { + g.Expect(filepath.Join(testStorage.BasePath, i.toPath)).Should(BeADirectory()) + } else { + g.Expect(filepath.Join(testStorage.BasePath, i.toPath)).ShouldNot(BeADirectory()) + } + } }) + } +} - type includeTestCase struct { - fromPath string - toPath string - createFiles []string - checkFiles []string +func TestGitRepositoryReconciler_reconcileDelete(t *testing.T) { + g := NewWithT(t) + + r := &GitRepositoryReconciler{ + Storage: testStorage, + } + + obj := &sourcev1.GitRepository{ + ObjectMeta: metav1.ObjectMeta{ + Name: "reconcile-delete-", + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Finalizers: []string{ + sourcev1.SourceFinalizer, + }, + }, + Status: sourcev1.GitRepositoryStatus{}, + } + + artifact := testStorage.NewArtifactFor(sourcev1.GitRepositoryKind, obj.GetObjectMeta(), "revision", "foo.txt") + obj.Status.Artifact = &artifact + + got, err := r.reconcileDelete(ctx, obj) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(got).To(Equal(ctrl.Result{})) + g.Expect(controllerutil.ContainsFinalizer(obj, sourcev1.SourceFinalizer)).To(BeFalse()) + g.Expect(obj.Status.Artifact).To(BeNil()) +} + +// func TestGitRepositoryReconciler_verifyCommitSignature(t *testing.T) { +// tests := []struct { +// name string +// secret *corev1.Secret +// commit git.Commit +// beforeFunc func(obj *sourcev1.GitRepository) +// want ctrl.Result +// wantErr bool +// assertConditions []metav1.Condition +// }{ +// { +// name: "Valid commit makes SourceVerifiedCondition=True", +// secret: &corev1.Secret{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "existing", +// }, +// }, +// commit: fake.NewCommit(true, "shasum"), +// beforeFunc: func(obj *sourcev1.GitRepository) { +// obj.Spec.Interval = metav1.Duration{Duration: interval} +// obj.Spec.Verification = &sourcev1.GitRepositoryVerification{ +// Mode: "head", +// SecretRef: meta.LocalObjectReference{ +// Name: "existing", +// }, +// } +// }, +// want: ctrl.Result{RequeueAfter: interval}, +// assertConditions: []metav1.Condition{ +// *conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "Verified signature of commit \"shasum\""), +// }, +// }, +// { +// name: "Invalid commit makes SourceVerifiedCondition=False and returns error", +// secret: &corev1.Secret{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "existing", +// }, +// }, +// commit: fake.NewCommit(false, "shasum"), +// beforeFunc: func(obj *sourcev1.GitRepository) { +// obj.Spec.Interval = metav1.Duration{Duration: interval} +// obj.Spec.Verification = &sourcev1.GitRepositoryVerification{ +// Mode: "head", +// SecretRef: meta.LocalObjectReference{ +// Name: "existing", +// }, +// } +// }, +// wantErr: true, +// assertConditions: []metav1.Condition{ +// *conditions.FalseCondition(sourcev1.SourceVerifiedCondition, meta.FailedReason, "Signature verification of commit \"shasum\" failed: invalid signature"), +// }, +// }, +// { +// name: "Secret get failure makes SourceVerified=False and returns error", +// beforeFunc: func(obj *sourcev1.GitRepository) { +// obj.Spec.Interval = metav1.Duration{Duration: interval} +// obj.Spec.Verification = &sourcev1.GitRepositoryVerification{ +// Mode: "head", +// SecretRef: meta.LocalObjectReference{ +// Name: "none-existing", +// }, +// } +// }, +// wantErr: true, +// assertConditions: []metav1.Condition{ +// *conditions.FalseCondition(sourcev1.SourceVerifiedCondition, meta.FailedReason, "PGP public keys secret error: secrets \"none-existing\" not found"), +// }, +// }, +// { +// name: "Nil verification in spec deletes SourceVerified condition", +// beforeFunc: func(obj *sourcev1.GitRepository) { +// obj.Spec.Interval = metav1.Duration{Duration: interval} +// conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, "Foo", "") +// }, +// want: ctrl.Result{RequeueAfter: interval}, +// assertConditions: []metav1.Condition{}, +// }, +// { +// name: "Empty verification mode in spec deletes SourceVerified condition", +// beforeFunc: func(obj *sourcev1.GitRepository) { +// obj.Spec.Interval = metav1.Duration{Duration: interval} +// obj.Spec.Verification = &sourcev1.GitRepositoryVerification{} +// conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, "Foo", "") +// }, +// want: ctrl.Result{RequeueAfter: interval}, +// assertConditions: []metav1.Condition{}, +// }, +// } + +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// g := NewWithT(t) + +// builder := fakeclient.NewClientBuilder().WithScheme(testEnv.GetScheme()) +// if tt.secret != nil { +// builder.WithObjects(tt.secret) +// } + +// r := &GitRepositoryReconciler{ +// Client: builder.Build(), +// } + +// obj := &sourcev1.GitRepository{ +// ObjectMeta: metav1.ObjectMeta{ +// GenerateName: "verify-commit-", +// Generation: 1, +// }, +// Status: sourcev1.GitRepositoryStatus{}, +// } + +// if tt.beforeFunc != nil { +// tt.beforeFunc(obj) +// } + +// got, err := r.verifyCommitSignature(logr.NewContext(ctx, log.NullLogger{}), obj, tt.commit) +// g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.assertConditions)) +// g.Expect(err != nil).To(Equal(tt.wantErr)) +// g.Expect(got).To(Equal(tt.want)) +// }) +// } +// } + +// helpers + +func initGitRepo(server *gittestserver.GitServer, fixture, branch, repositoryPath string) (*gogit.Repository, error) { + fs := memfs.New() + repo, err := gogit.Init(memory.NewStorage(), fs) + if err != nil { + return nil, err + } + + branchRef := plumbing.NewBranchReferenceName(branch) + if err = repo.CreateBranch(&config.Branch{ + Name: branch, + Remote: gogit.DefaultRemoteName, + Merge: branchRef, + }); err != nil { + return nil, err + } + + err = commitFromFixture(repo, fixture) + if err != nil { + return nil, err + } + + if server.HTTPAddress() == "" { + if err = server.StartHTTP(); err != nil { + return nil, err + } + defer server.StopHTTP() + } + if _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: gogit.DefaultRemoteName, + URLs: []string{server.HTTPAddressWithCredentials() + repositoryPath}, + }); err != nil { + return nil, err + } + + if err = repo.Push(&gogit.PushOptions{ + RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*"}, + }); err != nil { + return nil, err + } + + return repo, nil +} + +func Test_commitFromFixture(t *testing.T) { + g := NewWithT(t) + + repo, err := gogit.Init(memory.NewStorage(), memfs.New()) + g.Expect(err).ToNot(HaveOccurred()) + + err = commitFromFixture(repo, "testdata/git/repository") + g.Expect(err).ToNot(HaveOccurred()) +} + +func commitFromFixture(repo *gogit.Repository, fixture string) error { + working, err := repo.Worktree() + if err != nil { + return err + } + fs := working.Filesystem + + if err = filepath.Walk(fixture, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return fs.MkdirAll(fs.Join(path[len(fixture):]), info.Mode()) } - DescribeTable("Include git repositories", func(t includeTestCase) { - Expect(gitServer.StartHTTP()).To(Succeed()) - defer gitServer.StopHTTP() + fileBytes, err := os.ReadFile(path) + if err != nil { + return err + } - u, err := url.Parse(gitServer.HTTPAddress()) - Expect(err).NotTo(HaveOccurred()) + ff, err := fs.Create(path[len(fixture):]) + if err != nil { + return err + } + defer ff.Close() - // create the main git repository - mainRepoURL := *u - mainRepoURL.Path = path.Join(u.Path, fmt.Sprintf("repository-%s.git", randStringRunes(5))) + _, err = ff.Write(fileBytes) + return err + }); err != nil { + return err + } - mainFs := memfs.New() - mainRepo, err := git.Init(memory.NewStorage(), mainFs) - Expect(err).NotTo(HaveOccurred()) + _, err = working.Add(".") + if err != nil { + return err + } - mainWt, err := mainRepo.Worktree() - Expect(err).NotTo(HaveOccurred()) - - ff, _ := mainFs.Create("fixture") - _ = ff.Close() - _, err = mainWt.Add(mainFs.Join("fixture")) - Expect(err).NotTo(HaveOccurred()) - - _, err = mainWt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{ - Name: "John Doe", - Email: "john@example.com", - When: time.Now(), - }}) - Expect(err).NotTo(HaveOccurred()) - - mainRemote, err := mainRepo.CreateRemote(&config.RemoteConfig{ - Name: "origin", - URLs: []string{mainRepoURL.String()}, - }) - Expect(err).NotTo(HaveOccurred()) - - err = mainRemote.Push(&git.PushOptions{ - RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}, - }) - Expect(err).NotTo(HaveOccurred()) - - // create the sub git repository - subRepoURL := *u - subRepoURL.Path = path.Join(u.Path, fmt.Sprintf("subrepository-%s.git", randStringRunes(5))) - - subFs := memfs.New() - subRepo, err := git.Init(memory.NewStorage(), subFs) - Expect(err).NotTo(HaveOccurred()) - - subWt, err := subRepo.Worktree() - Expect(err).NotTo(HaveOccurred()) - - for _, v := range t.createFiles { - if dir := filepath.Base(v); dir != v { - err := subFs.MkdirAll(dir, 0700) - Expect(err).NotTo(HaveOccurred()) - } - ff, err := subFs.Create(v) - Expect(err).NotTo(HaveOccurred()) - _ = ff.Close() - _, err = subWt.Add(subFs.Join(v)) - Expect(err).NotTo(HaveOccurred()) - } - - _, err = subWt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{ - Name: "John Doe", - Email: "john@example.com", - When: time.Now(), - }}) - Expect(err).NotTo(HaveOccurred()) - - subRemote, err := subRepo.CreateRemote(&config.RemoteConfig{ - Name: "origin", - URLs: []string{subRepoURL.String()}, - }) - Expect(err).NotTo(HaveOccurred()) - - err = subRemote.Push(&git.PushOptions{ - RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}, - }) - Expect(err).NotTo(HaveOccurred()) - - // create main and sub resetRepositories - subKey := types.NamespacedName{ - Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)), - Namespace: namespace.Name, - } - subCreated := &sourcev1.GitRepository{ - ObjectMeta: metav1.ObjectMeta{ - Name: subKey.Name, - Namespace: subKey.Namespace, - }, - Spec: sourcev1.GitRepositorySpec{ - URL: subRepoURL.String(), - Interval: metav1.Duration{Duration: indexInterval}, - Reference: &sourcev1.GitRepositoryRef{Branch: "master"}, - }, - } - Expect(k8sClient.Create(context.Background(), subCreated)).Should(Succeed()) - defer k8sClient.Delete(context.Background(), subCreated) - - mainKey := types.NamespacedName{ - Name: fmt.Sprintf("git-ref-test-%s", randStringRunes(5)), - Namespace: namespace.Name, - } - mainCreated := &sourcev1.GitRepository{ - ObjectMeta: metav1.ObjectMeta{ - Name: mainKey.Name, - Namespace: mainKey.Namespace, - }, - Spec: sourcev1.GitRepositorySpec{ - URL: mainRepoURL.String(), - Interval: metav1.Duration{Duration: indexInterval}, - Reference: &sourcev1.GitRepositoryRef{Branch: "master"}, - Include: []sourcev1.GitRepositoryInclude{ - { - GitRepositoryRef: meta.LocalObjectReference{ - Name: subKey.Name, - }, - FromPath: t.fromPath, - ToPath: t.toPath, - }, - }, - }, - } - Expect(k8sClient.Create(context.Background(), mainCreated)).Should(Succeed()) - defer k8sClient.Delete(context.Background(), mainCreated) - - got := &sourcev1.GitRepository{} - Eventually(func() bool { - _ = k8sClient.Get(context.Background(), mainKey, got) - for _, c := range got.Status.Conditions { - if c.Reason == sourcev1.GitOperationSucceedReason { - return true - } - } - return false - }, timeout, interval).Should(BeTrue()) - - // check the contents of the repository - res, err := http.Get(got.Status.URL) - Expect(err).NotTo(HaveOccurred()) - Expect(res.StatusCode).To(Equal(http.StatusOK)) - tmp, err := os.MkdirTemp("", "flux-test") - Expect(err).NotTo(HaveOccurred()) - defer os.RemoveAll(tmp) - _, err = untar.Untar(res.Body, filepath.Join(tmp, "tar")) - Expect(err).NotTo(HaveOccurred()) - for _, v := range t.checkFiles { - Expect(filepath.Join(tmp, "tar", v)).To(BeAnExistingFile()) - } - - // add new file to check that the change is reconciled - ff, err = subFs.Create(subFs.Join(t.fromPath, "test")) - Expect(err).NotTo(HaveOccurred()) - err = ff.Close() - Expect(err).NotTo(HaveOccurred()) - _, err = subWt.Add(subFs.Join(t.fromPath, "test")) - Expect(err).NotTo(HaveOccurred()) - - hash, err := subWt.Commit("Sample", &git.CommitOptions{Author: &object.Signature{ - Name: "John Doe", - Email: "john@example.com", - When: time.Now(), - }}) - Expect(err).NotTo(HaveOccurred()) - - err = subRemote.Push(&git.PushOptions{ - RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}, - }) - Expect(err).NotTo(HaveOccurred()) - - got = &sourcev1.GitRepository{} - Eventually(func() bool { - _ = k8sClient.Get(context.Background(), mainKey, got) - if got.Status.IncludedArtifacts[0].Revision == fmt.Sprintf("master/%s", hash.String()) { - for _, c := range got.Status.Conditions { - if c.Reason == sourcev1.GitOperationSucceedReason { - return true - } - } - } - return false - }, timeout, interval).Should(BeTrue()) - - // get the main repository artifact - res, err = http.Get(got.Status.URL) - Expect(err).NotTo(HaveOccurred()) - Expect(res.StatusCode).To(Equal(http.StatusOK)) - tmp, err = os.MkdirTemp("", "flux-test") - Expect(err).NotTo(HaveOccurred()) - defer os.RemoveAll(tmp) - _, err = untar.Untar(res.Body, filepath.Join(tmp, "tar")) - Expect(err).NotTo(HaveOccurred()) - Expect(filepath.Join(tmp, "tar", t.toPath, "test")).To(BeAnExistingFile()) + if _, err = working.Commit("Fixtures from "+fixture, &gogit.CommitOptions{ + Author: &object.Signature{ + Name: "Jane Doe", + Email: "jane@example.com", + When: time.Now(), }, - Entry("only to path", includeTestCase{ - fromPath: "", - toPath: "sub", - createFiles: []string{"dir1", "dir2"}, - checkFiles: []string{"sub/dir1", "sub/dir2"}, - }), - Entry("to nested path", includeTestCase{ - fromPath: "", - toPath: "sub/nested", - createFiles: []string{"dir1", "dir2"}, - checkFiles: []string{"sub/nested/dir1", "sub/nested/dir2"}, - }), - Entry("from and to path", includeTestCase{ - fromPath: "nested", - toPath: "sub", - createFiles: []string{"dir1", "nested/dir2", "nested/dir3", "nested/foo/bar"}, - checkFiles: []string{"sub/dir2", "sub/dir3", "sub/foo/bar"}, - }), - ) + }); err != nil { + return err + } + + return nil +} + +func remoteBranchForHead(repo *gogit.Repository, head *plumbing.Reference, branch string) error { + refSpec := fmt.Sprintf("%s:refs/heads/%s", head.Name(), branch) + return repo.Push(&gogit.PushOptions{ + RemoteName: "origin", + RefSpecs: []config.RefSpec{config.RefSpec(refSpec)}, + Force: true, }) -}) +} + +func remoteTagForHead(repo *gogit.Repository, head *plumbing.Reference, tag string) error { + if _, err := repo.CreateTag(tag, head.Hash(), &gogit.CreateTagOptions{ + // Not setting this seems to make things flaky + // Expected success, but got an error: + // <*errors.errorString | 0xc0000f6350>: { + // s: "tagger field is required", + // } + // tagger field is required + Tagger: &object.Signature{ + Name: "Jane Doe", + Email: "jane@example.com", + When: time.Now(), + }, + Message: tag, + }); err != nil { + return err + } + refSpec := fmt.Sprintf("refs/tags/%[1]s:refs/tags/%[1]s", tag) + return repo.Push(&gogit.PushOptions{ + RefSpecs: []config.RefSpec{config.RefSpec(refSpec)}, + }) +} diff --git a/controllers/legacy_suite_test.go b/controllers/legacy_suite_test.go index 911f735b..748145fe 100644 --- a/controllers/legacy_suite_test.go +++ b/controllers/legacy_suite_test.go @@ -30,6 +30,7 @@ import ( "helm.sh/helm/v3/pkg/getter" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" @@ -118,11 +119,11 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) err = (&GitRepositoryReconciler{ - Client: k8sManager.GetClient(), - Scheme: scheme.Scheme, - Storage: ginkgoTestStorage, + Client: k8sManager.GetClient(), + EventRecorder: record.NewFakeRecorder(32), + Storage: ginkgoTestStorage, }).SetupWithManager(k8sManager) - Expect(err).ToNot(HaveOccurred(), "failed to setup GtRepositoryReconciler") + Expect(err).ToNot(HaveOccurred(), "failed to setup GitRepositoryReconciler") err = (&HelmRepositoryReconciler{ Client: k8sManager.GetClient(), diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 2710f6f7..a33108dc 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -26,6 +26,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "github.com/fluxcd/pkg/runtime/controller" @@ -87,13 +88,14 @@ func TestMain(m *testing.M) { testMetricsH = controller.MustMakeMetrics(testEnv) - //if err := (&GitRepositoryReconciler{ - // Client: testEnv, - // Metrics: testMetricsH, - // Storage: testStorage, - //}).SetupWithManager(testEnv); err != nil { - // panic(fmt.Sprintf("Failed to start GitRepositoryReconciler: %v", err)) - //} + if err := (&GitRepositoryReconciler{ + Client: testEnv, + EventRecorder: record.NewFakeRecorder(32), + Metrics: testMetricsH, + Storage: testStorage, + }).SetupWithManager(testEnv); err != nil { + panic(fmt.Sprintf("Failed to start GitRepositoryReconciler: %v", err)) + } go func() { fmt.Println("Starting the test environment") diff --git a/controllers/testdata/git/repository/.sourceignore b/controllers/testdata/git/repository/.sourceignore new file mode 100644 index 00000000..989478d1 --- /dev/null +++ b/controllers/testdata/git/repository/.sourceignore @@ -0,0 +1 @@ +**.txt diff --git a/controllers/testdata/git/repository/foo.txt b/controllers/testdata/git/repository/foo.txt new file mode 100644 index 00000000..e69de29b diff --git a/controllers/testdata/git/repository/manifest.yaml b/controllers/testdata/git/repository/manifest.yaml new file mode 100644 index 00000000..220e1b33 --- /dev/null +++ b/controllers/testdata/git/repository/manifest.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: dummy diff --git a/go.mod b/go.mod index 29496a95..dfae6782 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( k8s.io/api v0.23.1 k8s.io/apimachinery v0.23.1 k8s.io/client-go v0.23.1 + k8s.io/utils v0.0.0-20211208161948-7d6a63dca704 sigs.k8s.io/controller-runtime v0.11.0 sigs.k8s.io/yaml v1.3.0 ) @@ -200,7 +201,6 @@ require ( k8s.io/klog/v2 v2.30.0 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect k8s.io/kubectl v0.22.4 // indirect - k8s.io/utils v0.0.0-20211208161948-7d6a63dca704 // indirect oras.land/oras-go v0.4.0 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/kustomize/api v0.10.1 // indirect diff --git a/main.go b/main.go index bbced344..3c0f2791 100644 --- a/main.go +++ b/main.go @@ -165,11 +165,10 @@ func main() { storage := mustInitStorage(storagePath, storageAdvAddr, setupLog) if err = (&controllers.GitRepositoryReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Storage: storage, - EventRecorder: eventRecorder, - MetricsRecorder: metricsH.MetricsRecorder, + Client: mgr.GetClient(), + EventRecorder: eventRecorder, + Metrics: metricsH, + Storage: storage, }).SetupWithManagerAndOptions(mgr, controllers.GitRepositoryReconcilerOptions{ MaxConcurrentReconciles: concurrent, DependencyRequeueInterval: requeueDependency,