diff --git a/api/v1beta2/ocirepository_types.go b/api/v1beta2/ocirepository_types.go index 1aa855ac..b1a13508 100644 --- a/api/v1beta2/ocirepository_types.go +++ b/api/v1beta2/ocirepository_types.go @@ -45,6 +45,12 @@ const ( // AzureOCIProvider provides support for OCI authentication using a Azure Service Principal, // Managed Identity or Shared Key. AzureOCIProvider string = "azure" + + // OCILayerExtract defines the operation type for extracting the content from an OCI artifact layer. + OCILayerExtract = "extract" + + // OCILayerCopy defines the operation type for copying the content from an OCI artifact layer. + OCILayerCopy = "copy" ) // OCIRepositorySpec defines the desired state of OCIRepository @@ -156,6 +162,14 @@ type OCILayerSelector struct { // first layer matching this type is selected. // +optional MediaType string `json:"mediaType,omitempty"` + + // Operation specifies how the selected layer should be processed. + // By default, the layer compressed content is extracted to storage. + // When the operation is set to 'copy', the layer compressed content + // is persisted to storage as it is. + // +kubebuilder:validation:Enum=extract;copy + // +optional + Operation string `json:"operation,omitempty"` } // OCIRepositoryVerification verifies the authenticity of an OCI Artifact @@ -231,6 +245,15 @@ func (in *OCIRepository) GetLayerMediaType() string { return in.Spec.LayerSelector.MediaType } +// GetLayerOperation returns the layer selector operation (defaults to extract). +func (in *OCIRepository) GetLayerOperation() string { + if in.Spec.LayerSelector == nil || in.Spec.LayerSelector.Operation == "" { + return OCILayerExtract + } + + return in.Spec.LayerSelector.Operation +} + // +genclient // +genclient:Namespaced // +kubebuilder:storageversion diff --git a/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml index f4e94d19..a6c7ae40 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml @@ -90,6 +90,15 @@ spec: which should be extracted from the OCI Artifact. The first layer matching this type is selected. type: string + operation: + description: Operation specifies how the selected layer should + be processed. By default, the layer compressed content is extracted + to storage. When the operation is set to 'copy', the layer compressed + content is persisted to storage as it is. + enum: + - extract + - copy + type: string type: object provider: default: generic diff --git a/controllers/ocirepository_controller.go b/controllers/ocirepository_controller.go index 1003a574..95ec1ec6 100644 --- a/controllers/ocirepository_controller.go +++ b/controllers/ocirepository_controller.go @@ -22,8 +22,10 @@ import ( "crypto/x509" "errors" "fmt" + "io" "net/http" "os" + "path/filepath" "sort" "strings" "time" @@ -367,52 +369,23 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour return sreconcile.ResultEmpty, e } - // Pull artifact from the remote container registry - img, err := crane.Pull(url, options...) + // Get the upstream revision from the artifact digest + revision, err := r.getRevision(url, options) if err != nil { e := serror.NewGeneric( - fmt.Errorf("failed to pull artifact from '%s': %w", obj.Spec.URL, err), + fmt.Errorf("failed to determine artifact digest: %w", err), sourcev1.OCIPullFailedReason, ) conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) return sreconcile.ResultEmpty, e } - - // Determine the artifact SHA256 digest - imgDigest, err := img.Digest() - if err != nil { - e := serror.NewGeneric( - fmt.Errorf("failed to determine artifact digest: %w", err), - sourcev1.OCILayerOperationFailedReason, - ) - conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) - return sreconcile.ResultEmpty, e - } - - // Set the internal revision to the remote digest hex - revision := imgDigest.Hex - - // Copy the OCI annotations to the internal artifact metadata - manifest, err := img.Manifest() - if err != nil { - e := serror.NewGeneric( - fmt.Errorf("failed to parse artifact manifest: %w", err), - sourcev1.OCILayerOperationFailedReason, - ) - conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) - return sreconcile.ResultEmpty, e - } - - m := &sourcev1.Artifact{ - Revision: revision, - Metadata: manifest.Annotations, - } - m.DeepCopyInto(metadata) + metaArtifact := &sourcev1.Artifact{Revision: revision} + metaArtifact.DeepCopyInto(metadata) // Mark observations about the revision on the object defer func() { if !obj.GetArtifact().HasRevision(revision) { - message := fmt.Sprintf("new digest '%s' for '%s'", revision, url) + message := fmt.Sprintf("new revision '%s' for '%s'", revision, url) conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", message) conditions.MarkReconciling(obj, "NewRevision", message) } @@ -428,7 +401,7 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour } else if !obj.GetArtifact().HasRevision(revision) || conditions.GetObservedGeneration(obj, sourcev1.SourceVerifiedCondition) != obj.Generation || conditions.IsFalse(obj, sourcev1.SourceVerifiedCondition) { - err := r.verifyOCISourceSignature(ctx, obj, url, keychain) + err := r.verifySignature(ctx, obj, url, keychain) if err != nil { provider := obj.Spec.Verify.Provider if obj.Spec.Verify.SecretRef == nil { @@ -442,90 +415,177 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour return sreconcile.ResultEmpty, e } - conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of digest %s", revision) + conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of revision %s", revision) } - // Extract the content of the first artifact layer - if !obj.GetArtifact().HasRevision(revision) { - layers, err := img.Layers() - if err != nil { - e := serror.NewGeneric( - fmt.Errorf("failed to parse artifact layers: %w", err), - sourcev1.OCILayerOperationFailedReason, - ) - conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) - return sreconcile.ResultEmpty, e - } + // Skip pulling if the artifact revision hasn't changes + if obj.GetArtifact().HasRevision(revision) { + conditions.Delete(obj, sourcev1.FetchFailedCondition) + return sreconcile.ResultSuccess, nil + } - if len(layers) < 1 { - e := serror.NewGeneric( - fmt.Errorf("no layers found in artifact"), - sourcev1.OCILayerOperationFailedReason, - ) - conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) - return sreconcile.ResultEmpty, e - } + // Pull artifact from the remote container registry + img, err := crane.Pull(url, options...) + if err != nil { + e := serror.NewGeneric( + fmt.Errorf("failed to pull artifact from '%s': %w", obj.Spec.URL, err), + sourcev1.OCIPullFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e + } - var layer gcrv1.Layer + // Copy the OCI annotations to the internal artifact metadata + manifest, err := img.Manifest() + if err != nil { + e := serror.NewGeneric( + fmt.Errorf("failed to parse artifact manifest: %w", err), + sourcev1.OCILayerOperationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e + } + metadata.Metadata = manifest.Annotations - switch { - case obj.GetLayerMediaType() != "": - var found bool - for i, l := range layers { - md, err := l.MediaType() - if err != nil { - e := serror.NewGeneric( - fmt.Errorf("failed to determine the media type of layer[%v] from artifact: %w", i, err), - sourcev1.OCILayerOperationFailedReason, - ) - conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) - return sreconcile.ResultEmpty, e - } - if string(md) == obj.GetLayerMediaType() { - layer = layers[i] - found = true - break - } - } - if !found { - e := serror.NewGeneric( - fmt.Errorf("failed to find layer with media type '%s' in artifact", obj.GetLayerMediaType()), - sourcev1.OCILayerOperationFailedReason, - ) - conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) - return sreconcile.ResultEmpty, e - } - default: - layer = layers[0] - } - - blob, err := layer.Compressed() - if err != nil { - e := serror.NewGeneric( - fmt.Errorf("failed to extract the first layer from artifact: %w", err), - sourcev1.OCILayerOperationFailedReason, - ) - conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) - return sreconcile.ResultEmpty, e - } + // Extract the compressed content from the selected layer + blob, err := r.selectLayer(obj, img) + if err != nil { + e := serror.NewGeneric(err, sourcev1.OCILayerOperationFailedReason) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e + } + // Persist layer content to storage using the specified operation + switch obj.GetLayerOperation() { + case sourcev1.OCILayerExtract: if _, err = untar.Untar(blob, dir); err != nil { e := serror.NewGeneric( - fmt.Errorf("failed to untar the first layer from artifact: %w", err), + fmt.Errorf("failed to extract layer contents from artifact: %w", err), sourcev1.OCILayerOperationFailedReason, ) conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) return sreconcile.ResultEmpty, e } + case sourcev1.OCILayerCopy: + metadata.Path = fmt.Sprintf("%s.tgz", r.digestFromRevision(metadata.Revision)) + file, err := os.Create(filepath.Join(dir, metadata.Path)) + if err != nil { + e := serror.NewGeneric( + fmt.Errorf("failed to create file to copy layer to: %w", err), + sourcev1.OCILayerOperationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e + } + defer file.Close() + + _, err = io.Copy(file, blob) + if err != nil { + e := serror.NewGeneric( + fmt.Errorf("failed to copy layer from artifact: %w", err), + sourcev1.OCILayerOperationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e + } + default: + e := serror.NewGeneric( + fmt.Errorf("unsupported layer operation: %s", obj.GetLayerOperation()), + sourcev1.OCILayerOperationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e } conditions.Delete(obj, sourcev1.FetchFailedCondition) return sreconcile.ResultSuccess, nil } -// verifyOCISourceSignature verifies the authenticity of the given image reference url. First, it tries using a key +// selectLayer finds the matching layer and returns its compressed contents. +// If no layer selector was provided, we pick the first layer from the OCI artifact. +func (r *OCIRepositoryReconciler) selectLayer(obj *sourcev1.OCIRepository, image gcrv1.Image) (io.ReadCloser, error) { + layers, err := image.Layers() + if err != nil { + return nil, fmt.Errorf("failed to parse artifact layers: %w", err) + } + + if len(layers) < 1 { + return nil, fmt.Errorf("no layers found in artifact") + } + + var layer gcrv1.Layer + switch { + case obj.GetLayerMediaType() != "": + var found bool + for i, l := range layers { + md, err := l.MediaType() + if err != nil { + return nil, fmt.Errorf("failed to determine the media type of layer[%v] from artifact: %w", i, err) + } + if string(md) == obj.GetLayerMediaType() { + layer = layers[i] + found = true + break + } + } + if !found { + return nil, fmt.Errorf("failed to find layer with media type '%s' in artifact", obj.GetLayerMediaType()) + } + default: + layer = layers[0] + } + + blob, err := layer.Compressed() + if err != nil { + return nil, fmt.Errorf("failed to extract the first layer from artifact: %w", err) + } + + return blob, nil +} + +// getRevision fetches the upstream digest and returns the revision in the format `/` +func (r *OCIRepositoryReconciler) getRevision(url string, options []crane.Option) (string, error) { + ref, err := name.ParseReference(url) + if err != nil { + return "", err + } + + repoTag := "" + repoName := strings.TrimPrefix(url, ref.Context().RegistryStr()) + if s := strings.Split(repoName, ":"); len(s) == 2 && !strings.Contains(repoName, "@") { + repoTag = s[1] + } + + if repoTag == "" && !strings.Contains(repoName, "@") { + repoTag = "latest" + } + + digest, err := crane.Digest(url, options...) + if err != nil { + return "", err + } + + digestHash, err := gcrv1.NewHash(digest) + if err != nil { + return "", err + } + + revision := digestHash.Hex + if repoTag != "" { + revision = fmt.Sprintf("%s/%s", repoTag, digestHash.Hex) + } + return revision, nil +} + +// digestFromRevision extract the digest from the revision string +func (r *OCIRepositoryReconciler) digestFromRevision(revision string) string { + parts := strings.Split(revision, "/") + return parts[len(parts)-1] +} + +// verifySignature verifies the authenticity of the given image reference url. First, it tries using a key // if a secret with a valid public key is provided. If not, it falls back to a keyless approach for verification. -func (r *OCIRepositoryReconciler) verifyOCISourceSignature(ctx context.Context, obj *sourcev1.OCIRepository, url string, keychain authn.Keychain) error { +func (r *OCIRepositoryReconciler) verifySignature(ctx context.Context, obj *sourcev1.OCIRepository, url string, keychain authn.Keychain) error { ctxTimeout, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration) defer cancel() @@ -820,8 +880,7 @@ func (r *OCIRepositoryReconciler) craneOptions(ctx context.Context, insecure boo // condition is added. // The hostname of any URL in the Status of the object are updated, to ensure // they match the Storage server hostname of current runtime. -func (r *OCIRepositoryReconciler) reconcileStorage(ctx context.Context, - obj *sourcev1.OCIRepository, _ *sourcev1.Artifact, _ string) (sreconcile.Result, error) { +func (r *OCIRepositoryReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.OCIRepository, _ *sourcev1.Artifact, _ string) (sreconcile.Result, error) { // Garbage collect previous advertised artifact(s) from storage _ = r.garbageCollect(ctx, obj) @@ -856,13 +915,12 @@ func (r *OCIRepositoryReconciler) reconcileStorage(ctx context.Context, // early. // On a successful archive, the Artifact in the Status of the object is set, // and the symlink in the Storage is updated to its path. -func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context, - obj *sourcev1.OCIRepository, metadata *sourcev1.Artifact, dir string) (sreconcile.Result, error) { - // Calculate revision +func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.OCIRepository, metadata *sourcev1.Artifact, dir string) (sreconcile.Result, error) { revision := metadata.Revision // Create artifact - artifact := r.Storage.NewArtifactFor(obj.Kind, obj, revision, fmt.Sprintf("%s.tar.gz", revision)) + artifact := r.Storage.NewArtifactFor(obj.Kind, obj, revision, + fmt.Sprintf("%s.tar.gz", r.digestFromRevision(revision))) // Set the ArtifactInStorageCondition if there's no drift. defer func() { @@ -876,7 +934,7 @@ func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context, // The artifact is up-to-date if obj.GetArtifact().HasRevision(artifact.Revision) { r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason, - "artifact up-to-date with remote digest: '%s'", artifact.Revision) + "artifact up-to-date with remote revision: '%s'", artifact.Revision) return sreconcile.ResultSuccess, nil } @@ -915,14 +973,25 @@ func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context, } defer unlock() - // Archive directory to storage - if err := r.Storage.Archive(&artifact, dir, nil); err != nil { - e := serror.NewGeneric( - fmt.Errorf("unable to archive artifact to storage: %s", err), - sourcev1.ArchiveOperationFailedReason, - ) - conditions.MarkTrue(obj, sourcev1.StorageOperationFailedCondition, e.Reason, e.Err.Error()) - return sreconcile.ResultEmpty, e + switch obj.GetLayerOperation() { + case sourcev1.OCILayerCopy: + if err = r.Storage.CopyFromPath(&artifact, filepath.Join(dir, metadata.Path)); err != nil { + e := serror.NewGeneric( + fmt.Errorf("unable to copy artifact to storage: %w", err), + sourcev1.ArchiveOperationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.StorageOperationFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e + } + default: + if err := r.Storage.Archive(&artifact, dir, nil); err != nil { + e := serror.NewGeneric( + fmt.Errorf("unable to archive artifact to storage: %s", err), + sourcev1.ArchiveOperationFailedReason, + ) + conditions.MarkTrue(obj, sourcev1.StorageOperationFailedCondition, e.Reason, e.Err.Error()) + return sreconcile.ResultEmpty, e + } } // Record it on the object @@ -1000,8 +1069,7 @@ func (r *OCIRepositoryReconciler) garbageCollect(ctx context.Context, obj *sourc // This log is different from the debug log in the EventRecorder, in the sense // that this is a simple log. While the debug log contains complete details // about the event. -func (r *OCIRepositoryReconciler) eventLogf(ctx context.Context, - obj runtime.Object, eventType string, reason string, messageFmt string, args ...interface{}) { +func (r *OCIRepositoryReconciler) eventLogf(ctx context.Context, obj runtime.Object, eventType string, reason string, messageFmt string, args ...interface{}) { msg := fmt.Sprintf(messageFmt, args...) // Log and emit event. if eventType == corev1.EventTypeWarning { @@ -1013,8 +1081,7 @@ func (r *OCIRepositoryReconciler) eventLogf(ctx context.Context, } // notify emits notification related to the reconciliation. -func (r *OCIRepositoryReconciler) notify(ctx context.Context, - oldObj, newObj *sourcev1.OCIRepository, res sreconcile.Result, resErr error) { +func (r *OCIRepositoryReconciler) notify(ctx context.Context, oldObj, newObj *sourcev1.OCIRepository, res sreconcile.Result, resErr error) { // Notify successful reconciliation for new artifact and recovery from any // failure. if resErr == nil && res == sreconcile.ResultSuccess && newObj.Status.Artifact != nil { @@ -1028,7 +1095,7 @@ func (r *OCIRepositoryReconciler) notify(ctx context.Context, oldChecksum = oldObj.GetArtifact().Checksum } - message := fmt.Sprintf("stored artifact with digest '%s' from '%s'", newObj.Status.Artifact.Revision, newObj.Spec.URL) + message := fmt.Sprintf("stored artifact with revision '%s' from '%s'", newObj.Status.Artifact.Revision, newObj.Spec.URL) // enrich message with upstream annotations if found if info := newObj.GetArtifact().Metadata; info != nil { diff --git a/controllers/ocirepository_controller_test.go b/controllers/ocirepository_controller_test.go index 9bd4aa77..7449531c 100644 --- a/controllers/ocirepository_controller_test.go +++ b/controllers/ocirepository_controller_test.go @@ -85,14 +85,16 @@ func TestOCIRepository_Reconcile(t *testing.T) { semver string digest string mediaType string + operation string assertArtifact []artifactFixture }{ { name: "public tag", url: podinfoVersions["6.1.6"].url, tag: podinfoVersions["6.1.6"].tag, - digest: podinfoVersions["6.1.6"].digest.Hex, + digest: fmt.Sprintf("%s/%s", podinfoVersions["6.1.6"].tag, podinfoVersions["6.1.6"].digest.Hex), mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", + operation: sourcev1.OCILayerCopy, assertArtifact: []artifactFixture{ { expectedPath: "kustomize/deployment.yaml", @@ -108,7 +110,7 @@ func TestOCIRepository_Reconcile(t *testing.T) { name: "public semver", url: podinfoVersions["6.1.5"].url, semver: ">= 6.1 <= 6.1.5", - digest: podinfoVersions["6.1.5"].digest.Hex, + digest: fmt.Sprintf("%s/%s", podinfoVersions["6.1.5"].tag, podinfoVersions["6.1.5"].digest.Hex), assertArtifact: []artifactFixture{ { expectedPath: "kustomize/deployment.yaml", @@ -150,7 +152,12 @@ func TestOCIRepository_Reconcile(t *testing.T) { } if tt.mediaType != "" { obj.Spec.LayerSelector = &sourcev1.OCILayerSelector{MediaType: tt.mediaType} + + if tt.operation != "" { + obj.Spec.LayerSelector.Operation = tt.operation + } } + g.Expect(testEnv.Create(ctx, obj)).To(Succeed()) key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace} @@ -383,8 +390,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { name: "HTTP without basic auth", want: sreconcile.ResultSuccess, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), }, }, { @@ -404,8 +411,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { includeSecret: true, }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), }, }, { @@ -425,8 +432,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { includeSA: true, }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), }, }, { @@ -442,7 +449,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { }), }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact from "), + *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to determine artifact digest"), }, }, { @@ -463,7 +470,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { includeSecret: true, }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact from "), + *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "UNAUTHORIZED"), }, }, { @@ -484,7 +491,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { includeSA: true, }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact from "), + *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "UNAUTHORIZED"), }, }, { @@ -508,8 +515,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { }, }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), }, }, { @@ -526,7 +533,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { }), }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact from "), + *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to determine artifact digest"), }, }, { @@ -551,7 +558,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { }, }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact from "), + *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to determine artifact digest"), }, }, { @@ -580,8 +587,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { }, provider: "azure", assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), }, }, } @@ -676,7 +683,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { assertConditions := tt.assertConditions for k := range assertConditions { - assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", img.digest.Hex) + assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", fmt.Sprintf("%s/%s", img.tag, img.digest.Hex)) assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", repoURL) } @@ -864,10 +871,10 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { { name: "no reference (latest tag)", want: sreconcile.ResultSuccess, - wantRevision: img6.digest.Hex, + wantRevision: fmt.Sprintf("latest/%s", img6.digest.Hex), assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, { @@ -876,10 +883,10 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { Tag: "6.1.6", }, want: sreconcile.ResultSuccess, - wantRevision: img6.digest.Hex, + wantRevision: fmt.Sprintf("%s/%s", img6.tag, img6.digest.Hex), assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, { @@ -888,10 +895,10 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { SemVer: ">= 6.1.5", }, want: sreconcile.ResultSuccess, - wantRevision: img6.digest.Hex, + wantRevision: fmt.Sprintf("%s/%s", img6.tag, img6.digest.Hex), assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, { @@ -902,8 +909,8 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { wantRevision: img6.digest.Hex, want: sreconcile.ResultSuccess, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, { @@ -914,7 +921,7 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { want: sreconcile.ResultEmpty, wantErr: true, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact"), + *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, " MANIFEST_UNKNOWN"), }, }, { @@ -936,7 +943,7 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { want: sreconcile.ResultEmpty, wantErr: true, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact"), + *conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to determine artifact digest"), }, }, { @@ -946,10 +953,10 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { Tag: "6.1.5", }, want: sreconcile.ResultSuccess, - wantRevision: img6.digest.Hex, + wantRevision: fmt.Sprintf("%s/%s", img6.tag, img6.digest.Hex), assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, { @@ -962,8 +969,8 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { want: sreconcile.ResultSuccess, wantRevision: img5.digest.Hex, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, } @@ -1042,9 +1049,9 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { shouldSign: true, want: sreconcile.ResultSuccess, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of digest "), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of revision "), }, }, { @@ -1057,8 +1064,8 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { wantErrMsg: "failed to verify the signature using provider 'cosign': no matching signatures were found for ''", want: sreconcile.ResultEmpty, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), *conditions.FalseCondition(sourcev1.SourceVerifiedCondition, sourcev1.VerificationError, "failed to verify the signature using provider '': no matching signatures were found for ''"), }, }, @@ -1072,8 +1079,8 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { want: sreconcile.ResultEmpty, keyless: true, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), *conditions.FalseCondition(sourcev1.SourceVerifiedCondition, sourcev1.VerificationError, "failed to verify the signature using provider ' keyless': no matching signatures"), }, }, @@ -1084,7 +1091,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { beforeFunc: func(obj *sourcev1.OCIRepository) { conditions.MarkFalse(obj, sourcev1.SourceVerifiedCondition, "VerifyFailed", "fail msg") obj.Spec.Verify = nil - obj.Status.Artifact = &sourcev1.Artifact{Revision: img4.digest.Hex} + obj.Status.Artifact = &sourcev1.Artifact{Revision: fmt.Sprintf("%s/%s", img4.tag, img4.digest.Hex)} }, want: sreconcile.ResultSuccess, }, @@ -1094,7 +1101,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { digest: img4.digest.Hex, shouldSign: true, beforeFunc: func(obj *sourcev1.OCIRepository) { - obj.Status.Artifact = &sourcev1.Artifact{Revision: img4.digest.Hex} + obj.Status.Artifact = &sourcev1.Artifact{Revision: fmt.Sprintf("%s/%s", img4.tag, img4.digest.Hex)} // Set Verified with old observed generation and different reason/message. conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, "Verified", "verified") // Set new object generation. @@ -1102,7 +1109,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { }, want: sreconcile.ResultSuccess, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of digest "), + *conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of revision "), }, }, { @@ -1112,7 +1119,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { shouldSign: true, beforeFunc: func(obj *sourcev1.OCIRepository) { // Artifact present and custom verified condition reason/message. - obj.Status.Artifact = &sourcev1.Artifact{Revision: img4.digest.Hex} + obj.Status.Artifact = &sourcev1.Artifact{Revision: fmt.Sprintf("%s/%s", img4.tag, img4.digest.Hex)} conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, "Verified", "verified") }, want: sreconcile.ResultSuccess, @@ -1206,7 +1213,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { assertConditions := tt.assertConditions for k := range assertConditions { - assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", tt.digest) + assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", fmt.Sprintf("%s/%s", tt.reference.Tag, tt.digest)) assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", artifactURL) assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "", "cosign") } @@ -1251,7 +1258,7 @@ func TestOCIRepository_reconcileArtifact(t *testing.T) { Revision: "revision", }, beforeFunc: func(obj *sourcev1.OCIRepository) { - conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest") + conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision") }, want: sreconcile.ResultSuccess, assertPaths: []string{ @@ -1691,7 +1698,7 @@ func TestOCIRepositoryReconciler_notify(t *testing.T) { }, } }, - wantEvent: "Normal NewArtifact stored artifact with digest 'xxx' from 'oci://newurl.io', origin source 'https://github.com/stefanprodan/podinfo', origin revision '6.1.8/b3b00fe35424a45d373bf4c7214178bc36fd7872'", + wantEvent: "Normal NewArtifact stored artifact with revision 'xxx' from 'oci://newurl.io', origin source 'https://github.com/stefanprodan/podinfo', origin revision '6.1.8/b3b00fe35424a45d373bf4c7214178bc36fd7872'", }, { name: "recovery from failure", @@ -1707,7 +1714,7 @@ func TestOCIRepositoryReconciler_notify(t *testing.T) { obj.Status.Artifact = &sourcev1.Artifact{Revision: "xxx", Checksum: "yyy"} conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, "ready") }, - wantEvent: "Normal Succeeded stored artifact with digest 'xxx' from 'oci://newurl.io'", + wantEvent: "Normal Succeeded stored artifact with revision 'xxx' from 'oci://newurl.io'", }, { name: "recovery and new artifact", @@ -1723,7 +1730,7 @@ func TestOCIRepositoryReconciler_notify(t *testing.T) { obj.Status.Artifact = &sourcev1.Artifact{Revision: "aaa", Checksum: "bbb"} conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, "ready") }, - wantEvent: "Normal NewArtifact stored artifact with digest 'aaa' from 'oci://newurl.io'", + wantEvent: "Normal NewArtifact stored artifact with revision 'aaa' from 'oci://newurl.io'", }, { name: "no updates", diff --git a/docs/api/source.md b/docs/api/source.md index 9426f183..96b26b3e 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -2635,6 +2635,21 @@ which should be extracted from the OCI Artifact. The first layer matching this type is selected.

+ + +operation
+ +string + + + +(Optional) +

Operation specifies how the selected layer should be processed. +By default, the layer compressed content is extracted to storage. +When the operation is set to ‘copy’, the layer compressed content +is persisted to storage as it is.

+ + diff --git a/docs/spec/v1beta2/ocirepositories.md b/docs/spec/v1beta2/ocirepositories.md index 9e2e5069..76cc7386 100644 --- a/docs/spec/v1beta2/ocirepositories.md +++ b/docs/spec/v1beta2/ocirepositories.md @@ -31,7 +31,7 @@ In the above example: by the `.spec.interval` field. - It pulls the `latest` tag of the `ghcr.io/stefanprodan/manifests/podinfo` repository, indicated by the `.spec.ref.tag` and `.spec.url` fields. -- The resolved SHA256 digest is used as the Artifact +- The resolved tag and SHA256 digest is used as the Artifact revision, reported in-cluster in the `.status.artifact.revision` field. - When the current OCIRepository digest differs from the latest fetched digest, a new Artifact is archived. @@ -49,7 +49,7 @@ You can run this example by saving the manifest into `ocirepository.yaml`. ```console NAME URL AGE READY STATUS - podinfo oci://ghcr.io/stefanprodan/manifests/podinfo 5s True stored artifact with digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' + podinfo oci://ghcr.io/stefanprodan/manifests/podinfo 5s True stored artifact with revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' ``` 3. Run `kubectl describe ocirepository podinfo` to see the [Artifact](#artifact) @@ -62,17 +62,17 @@ You can run this example by saving the manifest into `ocirepository.yaml`. Checksum: d7e924b4882e55b97627355c7b3d2e711e9b54303afa2f50c25377f4df66a83b Last Update Time: 2022-06-14T11:23:36Z Path: ocirepository/default/podinfo/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de.tar.gz - Revision: 3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de + Revision: latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de URL: http://source-controller.flux-system.svc.cluster.local./ocirepository/oci/podinfo/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de.tar.gz Conditions: Last Transition Time: 2022-06-14T11:23:36Z - Message: stored artifact for digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' + Message: stored artifact for revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' Observed Generation: 1 Reason: Succeeded Status: True Type: Ready Last Transition Time: 2022-06-14T11:23:36Z - Message: stored artifact for digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' + Message: stored artifact for revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' Observed Generation: 1 Reason: Succeeded Status: True @@ -82,7 +82,7 @@ You can run this example by saving the manifest into `ocirepository.yaml`. Events: Type Reason Age From Message ---- ------ ---- ---- ------- - Normal NewArtifact 62s source-controller stored artifact with digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' from 'oci://ghcr.io/stefanprodan/manifests/podinfo' + Normal NewArtifact 62s source-controller stored artifact with revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' from 'oci://ghcr.io/stefanprodan/manifests/podinfo' ``` ## Writing an OCIRepository spec @@ -391,6 +391,7 @@ metadata: spec: layerSelector: mediaType: "application/deployment.content.v1.tar+gzip" + operation: extract # can be 'extract' or 'copy', defaults to 'extract' ``` If the layer selector matches more than one layer, the first layer matching the specified media type will be used. @@ -398,6 +399,10 @@ Note that the selected OCI layer must be [compressed](https://github.com/opencontainers/image-spec/blob/v1.0.2/layer.md#gzip-media-types) in the `tar+gzip` format. +When `.spec.layerSelector.operation` is set to `copy`, instead of extracting the +compressed layer, the controller copies the tarball as-is to storage, thus +keeping the original content unaltered. + ### Ignore `.spec.ignore` is an optional field to specify rules in [the `.gitignore` @@ -673,8 +678,8 @@ lists ```console LAST SEEN TYPE REASON OBJECT MESSAGE -2m14s Normal NewArtifact ocirepository/ stored artifact for digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' -36s Normal ArtifactUpToDate ocirepository/ artifact up-to-date with remote digest: '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' +2m14s Normal NewArtifact ocirepository/ stored artifact for revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' +36s Normal ArtifactUpToDate ocirepository/ artifact up-to-date with remote revision: 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' 94s Warning OCIOperationFailed ocirepository/ failed to pull artifact from 'oci://ghcr.io/stefanprodan/manifests/podinfo': couldn't find tag "0.0.1" ``` @@ -690,7 +695,7 @@ specific OCIRepository, e.g. The OCIRepository reports the latest synchronized state from the OCI repository as an Artifact object in the `.status.artifact` of the resource. -The `.status.artifact.revision` holds the SHA256 digest of the upstream OCI artifact. +The `.status.artifact.revision` holds the tag and SHA256 digest of the upstream OCI artifact. The `.status.artifact.metadata` holds the upstream OCI artifact metadata such as the [OpenContainers standard annotations](https://github.com/opencontainers/image-spec/blob/main/annotations.md). @@ -719,7 +724,7 @@ status: org.opencontainers.image.revision: 6.1.8/b3b00fe35424a45d373bf4c7214178bc36fd7872 org.opencontainers.image.source: https://github.com/stefanprodan/podinfo.git path: ocirepository///.tar.gz - revision: + revision: / url: http://source-controller..svc.cluster.local./ocirepository///.tar.gz ```