From 3f7d4630cc8e86646f19d3b464728bc4be2ac8a3 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Mon, 26 Sep 2022 13:05:27 +0300 Subject: [PATCH] Use the OCI artifact revision in status and events Signed-off-by: Stefan Prodan --- controllers/ocirepository_controller.go | 15 +++-- controllers/ocirepository_controller_test.go | 68 ++++++++++---------- docs/spec/v1beta2/ocirepositories.md | 25 ++++--- 3 files changed, 57 insertions(+), 51 deletions(-) diff --git a/controllers/ocirepository_controller.go b/controllers/ocirepository_controller.go index bd01a638..95ec1ec6 100644 --- a/controllers/ocirepository_controller.go +++ b/controllers/ocirepository_controller.go @@ -385,7 +385,7 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour // Mark observations about the revision on the object defer func() { if !obj.GetArtifact().HasRevision(revision) { - message := fmt.Sprintf("new digest '%s' for '%s'", revision, url) + message := fmt.Sprintf("new revision '%s' for '%s'", revision, url) conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", message) conditions.MarkReconciling(obj, "NewRevision", message) } @@ -415,7 +415,7 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour return sreconcile.ResultEmpty, e } - conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of digest %s", revision) + conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of revision %s", revision) } // Skip pulling if the artifact revision hasn't changes @@ -448,7 +448,7 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour metadata.Metadata = manifest.Annotations // Extract the compressed content from the selected layer - blob, err := r.getLayerCompressed(obj, img) + blob, err := r.selectLayer(obj, img) if err != nil { e := serror.NewGeneric(err, sourcev1.OCILayerOperationFailedReason) conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error()) @@ -501,8 +501,9 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour return sreconcile.ResultSuccess, nil } -// getLayerCompressed finds the matching layer and returns its compress contents -func (r *OCIRepositoryReconciler) getLayerCompressed(obj *sourcev1.OCIRepository, image gcrv1.Image) (io.ReadCloser, error) { +// selectLayer finds the matching layer and returns its compressed contents. +// If no layer selector was provided, we pick the first layer from the OCI artifact. +func (r *OCIRepositoryReconciler) selectLayer(obj *sourcev1.OCIRepository, image gcrv1.Image) (io.ReadCloser, error) { layers, err := image.Layers() if err != nil { return nil, fmt.Errorf("failed to parse artifact layers: %w", err) @@ -933,7 +934,7 @@ func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *so // The artifact is up-to-date if obj.GetArtifact().HasRevision(artifact.Revision) { r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason, - "artifact up-to-date with remote digest: '%s'", artifact.Revision) + "artifact up-to-date with remote revision: '%s'", artifact.Revision) return sreconcile.ResultSuccess, nil } @@ -1094,7 +1095,7 @@ func (r *OCIRepositoryReconciler) notify(ctx context.Context, oldObj, newObj *so oldChecksum = oldObj.GetArtifact().Checksum } - message := fmt.Sprintf("stored artifact with digest '%s' from '%s'", newObj.Status.Artifact.Revision, newObj.Spec.URL) + message := fmt.Sprintf("stored artifact with revision '%s' from '%s'", newObj.Status.Artifact.Revision, newObj.Spec.URL) // enrich message with upstream annotations if found if info := newObj.GetArtifact().Metadata; info != nil { diff --git a/controllers/ocirepository_controller_test.go b/controllers/ocirepository_controller_test.go index 82af8475..7449531c 100644 --- a/controllers/ocirepository_controller_test.go +++ b/controllers/ocirepository_controller_test.go @@ -390,8 +390,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { name: "HTTP without basic auth", want: sreconcile.ResultSuccess, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), }, }, { @@ -411,8 +411,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { includeSecret: true, }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), }, }, { @@ -432,8 +432,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { includeSA: true, }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), }, }, { @@ -515,8 +515,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { }, }, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), }, }, { @@ -587,8 +587,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) { }, provider: "azure", assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), }, }, } @@ -873,8 +873,8 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { want: sreconcile.ResultSuccess, wantRevision: fmt.Sprintf("latest/%s", img6.digest.Hex), assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, { @@ -885,8 +885,8 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { want: sreconcile.ResultSuccess, wantRevision: fmt.Sprintf("%s/%s", img6.tag, img6.digest.Hex), assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, { @@ -897,8 +897,8 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { want: sreconcile.ResultSuccess, wantRevision: fmt.Sprintf("%s/%s", img6.tag, img6.digest.Hex), assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, { @@ -909,8 +909,8 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { wantRevision: img6.digest.Hex, want: sreconcile.ResultSuccess, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, { @@ -955,8 +955,8 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { want: sreconcile.ResultSuccess, wantRevision: fmt.Sprintf("%s/%s", img6.tag, img6.digest.Hex), assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, { @@ -969,8 +969,8 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) { want: sreconcile.ResultSuccess, wantRevision: img5.digest.Hex, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"), }, }, } @@ -1049,9 +1049,9 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { shouldSign: true, want: sreconcile.ResultSuccess, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of digest "), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of revision "), }, }, { @@ -1064,8 +1064,8 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { wantErrMsg: "failed to verify the signature using provider 'cosign': no matching signatures were found for ''", want: sreconcile.ResultEmpty, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), *conditions.FalseCondition(sourcev1.SourceVerifiedCondition, sourcev1.VerificationError, "failed to verify the signature using provider '': no matching signatures were found for ''"), }, }, @@ -1079,8 +1079,8 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { want: sreconcile.ResultEmpty, keyless: true, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '' for ''"), - *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '' for ''"), + *conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '' for ''"), + *conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '' for ''"), *conditions.FalseCondition(sourcev1.SourceVerifiedCondition, sourcev1.VerificationError, "failed to verify the signature using provider ' keyless': no matching signatures"), }, }, @@ -1109,7 +1109,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) { }, want: sreconcile.ResultSuccess, assertConditions: []metav1.Condition{ - *conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of digest "), + *conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of revision "), }, }, { @@ -1258,7 +1258,7 @@ func TestOCIRepository_reconcileArtifact(t *testing.T) { Revision: "revision", }, beforeFunc: func(obj *sourcev1.OCIRepository) { - conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest") + conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision") }, want: sreconcile.ResultSuccess, assertPaths: []string{ @@ -1698,7 +1698,7 @@ func TestOCIRepositoryReconciler_notify(t *testing.T) { }, } }, - wantEvent: "Normal NewArtifact stored artifact with digest 'xxx' from 'oci://newurl.io', origin source 'https://github.com/stefanprodan/podinfo', origin revision '6.1.8/b3b00fe35424a45d373bf4c7214178bc36fd7872'", + wantEvent: "Normal NewArtifact stored artifact with revision 'xxx' from 'oci://newurl.io', origin source 'https://github.com/stefanprodan/podinfo', origin revision '6.1.8/b3b00fe35424a45d373bf4c7214178bc36fd7872'", }, { name: "recovery from failure", @@ -1714,7 +1714,7 @@ func TestOCIRepositoryReconciler_notify(t *testing.T) { obj.Status.Artifact = &sourcev1.Artifact{Revision: "xxx", Checksum: "yyy"} conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, "ready") }, - wantEvent: "Normal Succeeded stored artifact with digest 'xxx' from 'oci://newurl.io'", + wantEvent: "Normal Succeeded stored artifact with revision 'xxx' from 'oci://newurl.io'", }, { name: "recovery and new artifact", @@ -1730,7 +1730,7 @@ func TestOCIRepositoryReconciler_notify(t *testing.T) { obj.Status.Artifact = &sourcev1.Artifact{Revision: "aaa", Checksum: "bbb"} conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, "ready") }, - wantEvent: "Normal NewArtifact stored artifact with digest 'aaa' from 'oci://newurl.io'", + wantEvent: "Normal NewArtifact stored artifact with revision 'aaa' from 'oci://newurl.io'", }, { name: "no updates", diff --git a/docs/spec/v1beta2/ocirepositories.md b/docs/spec/v1beta2/ocirepositories.md index 9e2e5069..76cc7386 100644 --- a/docs/spec/v1beta2/ocirepositories.md +++ b/docs/spec/v1beta2/ocirepositories.md @@ -31,7 +31,7 @@ In the above example: by the `.spec.interval` field. - It pulls the `latest` tag of the `ghcr.io/stefanprodan/manifests/podinfo` repository, indicated by the `.spec.ref.tag` and `.spec.url` fields. -- The resolved SHA256 digest is used as the Artifact +- The resolved tag and SHA256 digest is used as the Artifact revision, reported in-cluster in the `.status.artifact.revision` field. - When the current OCIRepository digest differs from the latest fetched digest, a new Artifact is archived. @@ -49,7 +49,7 @@ You can run this example by saving the manifest into `ocirepository.yaml`. ```console NAME URL AGE READY STATUS - podinfo oci://ghcr.io/stefanprodan/manifests/podinfo 5s True stored artifact with digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' + podinfo oci://ghcr.io/stefanprodan/manifests/podinfo 5s True stored artifact with revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' ``` 3. Run `kubectl describe ocirepository podinfo` to see the [Artifact](#artifact) @@ -62,17 +62,17 @@ You can run this example by saving the manifest into `ocirepository.yaml`. Checksum: d7e924b4882e55b97627355c7b3d2e711e9b54303afa2f50c25377f4df66a83b Last Update Time: 2022-06-14T11:23:36Z Path: ocirepository/default/podinfo/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de.tar.gz - Revision: 3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de + Revision: latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de URL: http://source-controller.flux-system.svc.cluster.local./ocirepository/oci/podinfo/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de.tar.gz Conditions: Last Transition Time: 2022-06-14T11:23:36Z - Message: stored artifact for digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' + Message: stored artifact for revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' Observed Generation: 1 Reason: Succeeded Status: True Type: Ready Last Transition Time: 2022-06-14T11:23:36Z - Message: stored artifact for digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' + Message: stored artifact for revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' Observed Generation: 1 Reason: Succeeded Status: True @@ -82,7 +82,7 @@ You can run this example by saving the manifest into `ocirepository.yaml`. Events: Type Reason Age From Message ---- ------ ---- ---- ------- - Normal NewArtifact 62s source-controller stored artifact with digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' from 'oci://ghcr.io/stefanprodan/manifests/podinfo' + Normal NewArtifact 62s source-controller stored artifact with revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' from 'oci://ghcr.io/stefanprodan/manifests/podinfo' ``` ## Writing an OCIRepository spec @@ -391,6 +391,7 @@ metadata: spec: layerSelector: mediaType: "application/deployment.content.v1.tar+gzip" + operation: extract # can be 'extract' or 'copy', defaults to 'extract' ``` If the layer selector matches more than one layer, the first layer matching the specified media type will be used. @@ -398,6 +399,10 @@ Note that the selected OCI layer must be [compressed](https://github.com/opencontainers/image-spec/blob/v1.0.2/layer.md#gzip-media-types) in the `tar+gzip` format. +When `.spec.layerSelector.operation` is set to `copy`, instead of extracting the +compressed layer, the controller copies the tarball as-is to storage, thus +keeping the original content unaltered. + ### Ignore `.spec.ignore` is an optional field to specify rules in [the `.gitignore` @@ -673,8 +678,8 @@ lists ```console LAST SEEN TYPE REASON OBJECT MESSAGE -2m14s Normal NewArtifact ocirepository/ stored artifact for digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' -36s Normal ArtifactUpToDate ocirepository/ artifact up-to-date with remote digest: '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' +2m14s Normal NewArtifact ocirepository/ stored artifact for revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' +36s Normal ArtifactUpToDate ocirepository/ artifact up-to-date with remote revision: 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' 94s Warning OCIOperationFailed ocirepository/ failed to pull artifact from 'oci://ghcr.io/stefanprodan/manifests/podinfo': couldn't find tag "0.0.1" ``` @@ -690,7 +695,7 @@ specific OCIRepository, e.g. The OCIRepository reports the latest synchronized state from the OCI repository as an Artifact object in the `.status.artifact` of the resource. -The `.status.artifact.revision` holds the SHA256 digest of the upstream OCI artifact. +The `.status.artifact.revision` holds the tag and SHA256 digest of the upstream OCI artifact. The `.status.artifact.metadata` holds the upstream OCI artifact metadata such as the [OpenContainers standard annotations](https://github.com/opencontainers/image-spec/blob/main/annotations.md). @@ -719,7 +724,7 @@ status: org.opencontainers.image.revision: 6.1.8/b3b00fe35424a45d373bf4c7214178bc36fd7872 org.opencontainers.image.source: https://github.com/stefanprodan/podinfo.git path: ocirepository///.tar.gz - revision: + revision: / url: http://source-controller..svc.cluster.local./ocirepository///.tar.gz ```