Merge pull request #913 from fluxcd/oci-copy-layer

[OCIRepository] Optimise OCI artifacts reconciliation
This commit is contained in:
Stefan Prodan 2022-09-26 13:51:15 +03:00 committed by GitHub
commit 8bc36bcfc5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 308 additions and 182 deletions

View File

@ -45,6 +45,12 @@ const (
// AzureOCIProvider provides support for OCI authentication using a Azure Service Principal,
// Managed Identity or Shared Key.
AzureOCIProvider string = "azure"
// OCILayerExtract defines the operation type for extracting the content from an OCI artifact layer.
OCILayerExtract = "extract"
// OCILayerCopy defines the operation type for copying the content from an OCI artifact layer.
OCILayerCopy = "copy"
)
// OCIRepositorySpec defines the desired state of OCIRepository
@ -156,6 +162,14 @@ type OCILayerSelector struct {
// first layer matching this type is selected.
// +optional
MediaType string `json:"mediaType,omitempty"`
// Operation specifies how the selected layer should be processed.
// By default, the layer compressed content is extracted to storage.
// When the operation is set to 'copy', the layer compressed content
// is persisted to storage as it is.
// +kubebuilder:validation:Enum=extract;copy
// +optional
Operation string `json:"operation,omitempty"`
}
// OCIRepositoryVerification verifies the authenticity of an OCI Artifact
@ -231,6 +245,15 @@ func (in *OCIRepository) GetLayerMediaType() string {
return in.Spec.LayerSelector.MediaType
}
// GetLayerOperation returns the layer selector operation (defaults to extract).
func (in *OCIRepository) GetLayerOperation() string {
if in.Spec.LayerSelector == nil || in.Spec.LayerSelector.Operation == "" {
return OCILayerExtract
}
return in.Spec.LayerSelector.Operation
}
// +genclient
// +genclient:Namespaced
// +kubebuilder:storageversion

View File

@ -90,6 +90,15 @@ spec:
which should be extracted from the OCI Artifact. The first layer
matching this type is selected.
type: string
operation:
description: Operation specifies how the selected layer should
be processed. By default, the layer compressed content is extracted
to storage. When the operation is set to 'copy', the layer compressed
content is persisted to storage as it is.
enum:
- extract
- copy
type: string
type: object
provider:
default: generic

View File

@ -22,8 +22,10 @@ import (
"crypto/x509"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"time"
@ -367,52 +369,23 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour
return sreconcile.ResultEmpty, e
}
// Pull artifact from the remote container registry
img, err := crane.Pull(url, options...)
// Get the upstream revision from the artifact digest
revision, err := r.getRevision(url, options)
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to pull artifact from '%s': %w", obj.Spec.URL, err),
fmt.Errorf("failed to determine artifact digest: %w", err),
sourcev1.OCIPullFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
// Determine the artifact SHA256 digest
imgDigest, err := img.Digest()
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to determine artifact digest: %w", err),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
// Set the internal revision to the remote digest hex
revision := imgDigest.Hex
// Copy the OCI annotations to the internal artifact metadata
manifest, err := img.Manifest()
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to parse artifact manifest: %w", err),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
m := &sourcev1.Artifact{
Revision: revision,
Metadata: manifest.Annotations,
}
m.DeepCopyInto(metadata)
metaArtifact := &sourcev1.Artifact{Revision: revision}
metaArtifact.DeepCopyInto(metadata)
// Mark observations about the revision on the object
defer func() {
if !obj.GetArtifact().HasRevision(revision) {
message := fmt.Sprintf("new digest '%s' for '%s'", revision, url)
message := fmt.Sprintf("new revision '%s' for '%s'", revision, url)
conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", message)
conditions.MarkReconciling(obj, "NewRevision", message)
}
@ -428,7 +401,7 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour
} else if !obj.GetArtifact().HasRevision(revision) ||
conditions.GetObservedGeneration(obj, sourcev1.SourceVerifiedCondition) != obj.Generation ||
conditions.IsFalse(obj, sourcev1.SourceVerifiedCondition) {
err := r.verifyOCISourceSignature(ctx, obj, url, keychain)
err := r.verifySignature(ctx, obj, url, keychain)
if err != nil {
provider := obj.Spec.Verify.Provider
if obj.Spec.Verify.SecretRef == nil {
@ -442,90 +415,177 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour
return sreconcile.ResultEmpty, e
}
conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of digest %s", revision)
conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of revision %s", revision)
}
// Extract the content of the first artifact layer
if !obj.GetArtifact().HasRevision(revision) {
layers, err := img.Layers()
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to parse artifact layers: %w", err),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
// Skip pulling if the artifact revision hasn't changes
if obj.GetArtifact().HasRevision(revision) {
conditions.Delete(obj, sourcev1.FetchFailedCondition)
return sreconcile.ResultSuccess, nil
}
if len(layers) < 1 {
e := serror.NewGeneric(
fmt.Errorf("no layers found in artifact"),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
// Pull artifact from the remote container registry
img, err := crane.Pull(url, options...)
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to pull artifact from '%s': %w", obj.Spec.URL, err),
sourcev1.OCIPullFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
var layer gcrv1.Layer
// Copy the OCI annotations to the internal artifact metadata
manifest, err := img.Manifest()
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to parse artifact manifest: %w", err),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
metadata.Metadata = manifest.Annotations
switch {
case obj.GetLayerMediaType() != "":
var found bool
for i, l := range layers {
md, err := l.MediaType()
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to determine the media type of layer[%v] from artifact: %w", i, err),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
if string(md) == obj.GetLayerMediaType() {
layer = layers[i]
found = true
break
}
}
if !found {
e := serror.NewGeneric(
fmt.Errorf("failed to find layer with media type '%s' in artifact", obj.GetLayerMediaType()),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
default:
layer = layers[0]
}
blob, err := layer.Compressed()
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to extract the first layer from artifact: %w", err),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
// Extract the compressed content from the selected layer
blob, err := r.selectLayer(obj, img)
if err != nil {
e := serror.NewGeneric(err, sourcev1.OCILayerOperationFailedReason)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
// Persist layer content to storage using the specified operation
switch obj.GetLayerOperation() {
case sourcev1.OCILayerExtract:
if _, err = untar.Untar(blob, dir); err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to untar the first layer from artifact: %w", err),
fmt.Errorf("failed to extract layer contents from artifact: %w", err),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
case sourcev1.OCILayerCopy:
metadata.Path = fmt.Sprintf("%s.tgz", r.digestFromRevision(metadata.Revision))
file, err := os.Create(filepath.Join(dir, metadata.Path))
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to create file to copy layer to: %w", err),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
defer file.Close()
_, err = io.Copy(file, blob)
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to copy layer from artifact: %w", err),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
default:
e := serror.NewGeneric(
fmt.Errorf("unsupported layer operation: %s", obj.GetLayerOperation()),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
conditions.Delete(obj, sourcev1.FetchFailedCondition)
return sreconcile.ResultSuccess, nil
}
// verifyOCISourceSignature verifies the authenticity of the given image reference url. First, it tries using a key
// selectLayer finds the matching layer and returns its compressed contents.
// If no layer selector was provided, we pick the first layer from the OCI artifact.
func (r *OCIRepositoryReconciler) selectLayer(obj *sourcev1.OCIRepository, image gcrv1.Image) (io.ReadCloser, error) {
layers, err := image.Layers()
if err != nil {
return nil, fmt.Errorf("failed to parse artifact layers: %w", err)
}
if len(layers) < 1 {
return nil, fmt.Errorf("no layers found in artifact")
}
var layer gcrv1.Layer
switch {
case obj.GetLayerMediaType() != "":
var found bool
for i, l := range layers {
md, err := l.MediaType()
if err != nil {
return nil, fmt.Errorf("failed to determine the media type of layer[%v] from artifact: %w", i, err)
}
if string(md) == obj.GetLayerMediaType() {
layer = layers[i]
found = true
break
}
}
if !found {
return nil, fmt.Errorf("failed to find layer with media type '%s' in artifact", obj.GetLayerMediaType())
}
default:
layer = layers[0]
}
blob, err := layer.Compressed()
if err != nil {
return nil, fmt.Errorf("failed to extract the first layer from artifact: %w", err)
}
return blob, nil
}
// getRevision fetches the upstream digest and returns the revision in the format `<tag>/<digest>`
func (r *OCIRepositoryReconciler) getRevision(url string, options []crane.Option) (string, error) {
ref, err := name.ParseReference(url)
if err != nil {
return "", err
}
repoTag := ""
repoName := strings.TrimPrefix(url, ref.Context().RegistryStr())
if s := strings.Split(repoName, ":"); len(s) == 2 && !strings.Contains(repoName, "@") {
repoTag = s[1]
}
if repoTag == "" && !strings.Contains(repoName, "@") {
repoTag = "latest"
}
digest, err := crane.Digest(url, options...)
if err != nil {
return "", err
}
digestHash, err := gcrv1.NewHash(digest)
if err != nil {
return "", err
}
revision := digestHash.Hex
if repoTag != "" {
revision = fmt.Sprintf("%s/%s", repoTag, digestHash.Hex)
}
return revision, nil
}
// digestFromRevision extract the digest from the revision string
func (r *OCIRepositoryReconciler) digestFromRevision(revision string) string {
parts := strings.Split(revision, "/")
return parts[len(parts)-1]
}
// verifySignature verifies the authenticity of the given image reference url. First, it tries using a key
// if a secret with a valid public key is provided. If not, it falls back to a keyless approach for verification.
func (r *OCIRepositoryReconciler) verifyOCISourceSignature(ctx context.Context, obj *sourcev1.OCIRepository, url string, keychain authn.Keychain) error {
func (r *OCIRepositoryReconciler) verifySignature(ctx context.Context, obj *sourcev1.OCIRepository, url string, keychain authn.Keychain) error {
ctxTimeout, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
defer cancel()
@ -820,8 +880,7 @@ func (r *OCIRepositoryReconciler) craneOptions(ctx context.Context, insecure boo
// condition is added.
// The hostname of any URL in the Status of the object are updated, to ensure
// they match the Storage server hostname of current runtime.
func (r *OCIRepositoryReconciler) reconcileStorage(ctx context.Context,
obj *sourcev1.OCIRepository, _ *sourcev1.Artifact, _ string) (sreconcile.Result, error) {
func (r *OCIRepositoryReconciler) reconcileStorage(ctx context.Context, obj *sourcev1.OCIRepository, _ *sourcev1.Artifact, _ string) (sreconcile.Result, error) {
// Garbage collect previous advertised artifact(s) from storage
_ = r.garbageCollect(ctx, obj)
@ -856,13 +915,12 @@ func (r *OCIRepositoryReconciler) reconcileStorage(ctx context.Context,
// early.
// On a successful archive, the Artifact in the Status of the object is set,
// and the symlink in the Storage is updated to its path.
func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context,
obj *sourcev1.OCIRepository, metadata *sourcev1.Artifact, dir string) (sreconcile.Result, error) {
// Calculate revision
func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.OCIRepository, metadata *sourcev1.Artifact, dir string) (sreconcile.Result, error) {
revision := metadata.Revision
// Create artifact
artifact := r.Storage.NewArtifactFor(obj.Kind, obj, revision, fmt.Sprintf("%s.tar.gz", revision))
artifact := r.Storage.NewArtifactFor(obj.Kind, obj, revision,
fmt.Sprintf("%s.tar.gz", r.digestFromRevision(revision)))
// Set the ArtifactInStorageCondition if there's no drift.
defer func() {
@ -876,7 +934,7 @@ func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context,
// The artifact is up-to-date
if obj.GetArtifact().HasRevision(artifact.Revision) {
r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason,
"artifact up-to-date with remote digest: '%s'", artifact.Revision)
"artifact up-to-date with remote revision: '%s'", artifact.Revision)
return sreconcile.ResultSuccess, nil
}
@ -915,14 +973,25 @@ func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context,
}
defer unlock()
// Archive directory to storage
if err := r.Storage.Archive(&artifact, dir, nil); err != nil {
e := serror.NewGeneric(
fmt.Errorf("unable to archive artifact to storage: %s", err),
sourcev1.ArchiveOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.StorageOperationFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
switch obj.GetLayerOperation() {
case sourcev1.OCILayerCopy:
if err = r.Storage.CopyFromPath(&artifact, filepath.Join(dir, metadata.Path)); err != nil {
e := serror.NewGeneric(
fmt.Errorf("unable to copy artifact to storage: %w", err),
sourcev1.ArchiveOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.StorageOperationFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
default:
if err := r.Storage.Archive(&artifact, dir, nil); err != nil {
e := serror.NewGeneric(
fmt.Errorf("unable to archive artifact to storage: %s", err),
sourcev1.ArchiveOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.StorageOperationFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
}
// Record it on the object
@ -1000,8 +1069,7 @@ func (r *OCIRepositoryReconciler) garbageCollect(ctx context.Context, obj *sourc
// This log is different from the debug log in the EventRecorder, in the sense
// that this is a simple log. While the debug log contains complete details
// about the event.
func (r *OCIRepositoryReconciler) eventLogf(ctx context.Context,
obj runtime.Object, eventType string, reason string, messageFmt string, args ...interface{}) {
func (r *OCIRepositoryReconciler) eventLogf(ctx context.Context, obj runtime.Object, eventType string, reason string, messageFmt string, args ...interface{}) {
msg := fmt.Sprintf(messageFmt, args...)
// Log and emit event.
if eventType == corev1.EventTypeWarning {
@ -1013,8 +1081,7 @@ func (r *OCIRepositoryReconciler) eventLogf(ctx context.Context,
}
// notify emits notification related to the reconciliation.
func (r *OCIRepositoryReconciler) notify(ctx context.Context,
oldObj, newObj *sourcev1.OCIRepository, res sreconcile.Result, resErr error) {
func (r *OCIRepositoryReconciler) notify(ctx context.Context, oldObj, newObj *sourcev1.OCIRepository, res sreconcile.Result, resErr error) {
// Notify successful reconciliation for new artifact and recovery from any
// failure.
if resErr == nil && res == sreconcile.ResultSuccess && newObj.Status.Artifact != nil {
@ -1028,7 +1095,7 @@ func (r *OCIRepositoryReconciler) notify(ctx context.Context,
oldChecksum = oldObj.GetArtifact().Checksum
}
message := fmt.Sprintf("stored artifact with digest '%s' from '%s'", newObj.Status.Artifact.Revision, newObj.Spec.URL)
message := fmt.Sprintf("stored artifact with revision '%s' from '%s'", newObj.Status.Artifact.Revision, newObj.Spec.URL)
// enrich message with upstream annotations if found
if info := newObj.GetArtifact().Metadata; info != nil {

View File

@ -85,14 +85,16 @@ func TestOCIRepository_Reconcile(t *testing.T) {
semver string
digest string
mediaType string
operation string
assertArtifact []artifactFixture
}{
{
name: "public tag",
url: podinfoVersions["6.1.6"].url,
tag: podinfoVersions["6.1.6"].tag,
digest: podinfoVersions["6.1.6"].digest.Hex,
digest: fmt.Sprintf("%s/%s", podinfoVersions["6.1.6"].tag, podinfoVersions["6.1.6"].digest.Hex),
mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
operation: sourcev1.OCILayerCopy,
assertArtifact: []artifactFixture{
{
expectedPath: "kustomize/deployment.yaml",
@ -108,7 +110,7 @@ func TestOCIRepository_Reconcile(t *testing.T) {
name: "public semver",
url: podinfoVersions["6.1.5"].url,
semver: ">= 6.1 <= 6.1.5",
digest: podinfoVersions["6.1.5"].digest.Hex,
digest: fmt.Sprintf("%s/%s", podinfoVersions["6.1.5"].tag, podinfoVersions["6.1.5"].digest.Hex),
assertArtifact: []artifactFixture{
{
expectedPath: "kustomize/deployment.yaml",
@ -150,7 +152,12 @@ func TestOCIRepository_Reconcile(t *testing.T) {
}
if tt.mediaType != "" {
obj.Spec.LayerSelector = &sourcev1.OCILayerSelector{MediaType: tt.mediaType}
if tt.operation != "" {
obj.Spec.LayerSelector.Operation = tt.operation
}
}
g.Expect(testEnv.Create(ctx, obj)).To(Succeed())
key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}
@ -383,8 +390,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
name: "HTTP without basic auth",
want: sreconcile.ResultSuccess,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
},
},
{
@ -404,8 +411,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
includeSecret: true,
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
},
},
{
@ -425,8 +432,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
includeSA: true,
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
},
},
{
@ -442,7 +449,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
}),
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact from "),
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to determine artifact digest"),
},
},
{
@ -463,7 +470,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
includeSecret: true,
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact from "),
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "UNAUTHORIZED"),
},
},
{
@ -484,7 +491,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
includeSA: true,
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact from "),
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "UNAUTHORIZED"),
},
},
{
@ -508,8 +515,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
},
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
},
},
{
@ -526,7 +533,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
}),
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact from "),
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to determine artifact digest"),
},
},
{
@ -551,7 +558,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
},
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact from "),
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to determine artifact digest"),
},
},
{
@ -580,8 +587,8 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
},
provider: "azure",
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
},
},
}
@ -676,7 +683,7 @@ func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
assertConditions := tt.assertConditions
for k := range assertConditions {
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<digest>", img.digest.Hex)
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<digest>", fmt.Sprintf("%s/%s", img.tag, img.digest.Hex))
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<url>", repoURL)
}
@ -864,10 +871,10 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) {
{
name: "no reference (latest tag)",
want: sreconcile.ResultSuccess,
wantRevision: img6.digest.Hex,
wantRevision: fmt.Sprintf("latest/%s", img6.digest.Hex),
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"),
},
},
{
@ -876,10 +883,10 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) {
Tag: "6.1.6",
},
want: sreconcile.ResultSuccess,
wantRevision: img6.digest.Hex,
wantRevision: fmt.Sprintf("%s/%s", img6.tag, img6.digest.Hex),
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"),
},
},
{
@ -888,10 +895,10 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) {
SemVer: ">= 6.1.5",
},
want: sreconcile.ResultSuccess,
wantRevision: img6.digest.Hex,
wantRevision: fmt.Sprintf("%s/%s", img6.tag, img6.digest.Hex),
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"),
},
},
{
@ -902,8 +909,8 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) {
wantRevision: img6.digest.Hex,
want: sreconcile.ResultSuccess,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"),
},
},
{
@ -914,7 +921,7 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) {
want: sreconcile.ResultEmpty,
wantErr: true,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact"),
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, " MANIFEST_UNKNOWN"),
},
},
{
@ -936,7 +943,7 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) {
want: sreconcile.ResultEmpty,
wantErr: true,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to pull artifact"),
*conditions.TrueCondition(sourcev1.FetchFailedCondition, sourcev1.OCIPullFailedReason, "failed to determine artifact digest"),
},
},
{
@ -946,10 +953,10 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) {
Tag: "6.1.5",
},
want: sreconcile.ResultSuccess,
wantRevision: img6.digest.Hex,
wantRevision: fmt.Sprintf("%s/%s", img6.tag, img6.digest.Hex),
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"),
},
},
{
@ -962,8 +969,8 @@ func TestOCIRepository_reconcileSource_remoteReference(t *testing.T) {
want: sreconcile.ResultSuccess,
wantRevision: img5.digest.Hex,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision"),
},
},
}
@ -1042,9 +1049,9 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) {
shouldSign: true,
want: sreconcile.ResultSuccess,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of digest <digest>"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of revision <digest>"),
},
},
{
@ -1057,8 +1064,8 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) {
wantErrMsg: "failed to verify the signature using provider 'cosign': no matching signatures were found for '<url>'",
want: sreconcile.ResultEmpty,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.FalseCondition(sourcev1.SourceVerifiedCondition, sourcev1.VerificationError, "failed to verify the signature using provider '<provider>': no matching signatures were found for '<url>'"),
},
},
@ -1072,8 +1079,8 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) {
want: sreconcile.ResultEmpty,
keyless: true,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest '<digest>' for '<url>'"),
*conditions.TrueCondition(meta.ReconcilingCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.TrueCondition(sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision '<digest>' for '<url>'"),
*conditions.FalseCondition(sourcev1.SourceVerifiedCondition, sourcev1.VerificationError, "failed to verify the signature using provider '<provider> keyless': no matching signatures"),
},
},
@ -1084,7 +1091,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) {
beforeFunc: func(obj *sourcev1.OCIRepository) {
conditions.MarkFalse(obj, sourcev1.SourceVerifiedCondition, "VerifyFailed", "fail msg")
obj.Spec.Verify = nil
obj.Status.Artifact = &sourcev1.Artifact{Revision: img4.digest.Hex}
obj.Status.Artifact = &sourcev1.Artifact{Revision: fmt.Sprintf("%s/%s", img4.tag, img4.digest.Hex)}
},
want: sreconcile.ResultSuccess,
},
@ -1094,7 +1101,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) {
digest: img4.digest.Hex,
shouldSign: true,
beforeFunc: func(obj *sourcev1.OCIRepository) {
obj.Status.Artifact = &sourcev1.Artifact{Revision: img4.digest.Hex}
obj.Status.Artifact = &sourcev1.Artifact{Revision: fmt.Sprintf("%s/%s", img4.tag, img4.digest.Hex)}
// Set Verified with old observed generation and different reason/message.
conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, "Verified", "verified")
// Set new object generation.
@ -1102,7 +1109,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) {
},
want: sreconcile.ResultSuccess,
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of digest <digest>"),
*conditions.TrueCondition(sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of revision <digest>"),
},
},
{
@ -1112,7 +1119,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) {
shouldSign: true,
beforeFunc: func(obj *sourcev1.OCIRepository) {
// Artifact present and custom verified condition reason/message.
obj.Status.Artifact = &sourcev1.Artifact{Revision: img4.digest.Hex}
obj.Status.Artifact = &sourcev1.Artifact{Revision: fmt.Sprintf("%s/%s", img4.tag, img4.digest.Hex)}
conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, "Verified", "verified")
},
want: sreconcile.ResultSuccess,
@ -1206,7 +1213,7 @@ func TestOCIRepository_reconcileSource_verifyOCISourceSignature(t *testing.T) {
assertConditions := tt.assertConditions
for k := range assertConditions {
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<digest>", tt.digest)
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<digest>", fmt.Sprintf("%s/%s", tt.reference.Tag, tt.digest))
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<url>", artifactURL)
assertConditions[k].Message = strings.ReplaceAll(assertConditions[k].Message, "<provider>", "cosign")
}
@ -1251,7 +1258,7 @@ func TestOCIRepository_reconcileArtifact(t *testing.T) {
Revision: "revision",
},
beforeFunc: func(obj *sourcev1.OCIRepository) {
conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", "new digest")
conditions.MarkTrue(obj, sourcev1.ArtifactOutdatedCondition, "NewRevision", "new revision")
},
want: sreconcile.ResultSuccess,
assertPaths: []string{
@ -1691,7 +1698,7 @@ func TestOCIRepositoryReconciler_notify(t *testing.T) {
},
}
},
wantEvent: "Normal NewArtifact stored artifact with digest 'xxx' from 'oci://newurl.io', origin source 'https://github.com/stefanprodan/podinfo', origin revision '6.1.8/b3b00fe35424a45d373bf4c7214178bc36fd7872'",
wantEvent: "Normal NewArtifact stored artifact with revision 'xxx' from 'oci://newurl.io', origin source 'https://github.com/stefanprodan/podinfo', origin revision '6.1.8/b3b00fe35424a45d373bf4c7214178bc36fd7872'",
},
{
name: "recovery from failure",
@ -1707,7 +1714,7 @@ func TestOCIRepositoryReconciler_notify(t *testing.T) {
obj.Status.Artifact = &sourcev1.Artifact{Revision: "xxx", Checksum: "yyy"}
conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, "ready")
},
wantEvent: "Normal Succeeded stored artifact with digest 'xxx' from 'oci://newurl.io'",
wantEvent: "Normal Succeeded stored artifact with revision 'xxx' from 'oci://newurl.io'",
},
{
name: "recovery and new artifact",
@ -1723,7 +1730,7 @@ func TestOCIRepositoryReconciler_notify(t *testing.T) {
obj.Status.Artifact = &sourcev1.Artifact{Revision: "aaa", Checksum: "bbb"}
conditions.MarkTrue(obj, meta.ReadyCondition, meta.SucceededReason, "ready")
},
wantEvent: "Normal NewArtifact stored artifact with digest 'aaa' from 'oci://newurl.io'",
wantEvent: "Normal NewArtifact stored artifact with revision 'aaa' from 'oci://newurl.io'",
},
{
name: "no updates",

View File

@ -2635,6 +2635,21 @@ which should be extracted from the OCI Artifact. The
first layer matching this type is selected.</p>
</td>
</tr>
<tr>
<td>
<code>operation</code><br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>Operation specifies how the selected layer should be processed.
By default, the layer compressed content is extracted to storage.
When the operation is set to &lsquo;copy&rsquo;, the layer compressed content
is persisted to storage as it is.</p>
</td>
</tr>
</tbody>
</table>
</div>

View File

@ -31,7 +31,7 @@ In the above example:
by the `.spec.interval` field.
- It pulls the `latest` tag of the `ghcr.io/stefanprodan/manifests/podinfo`
repository, indicated by the `.spec.ref.tag` and `.spec.url` fields.
- The resolved SHA256 digest is used as the Artifact
- The resolved tag and SHA256 digest is used as the Artifact
revision, reported in-cluster in the `.status.artifact.revision` field.
- When the current OCIRepository digest differs from the latest fetched
digest, a new Artifact is archived.
@ -49,7 +49,7 @@ You can run this example by saving the manifest into `ocirepository.yaml`.
```console
NAME URL AGE READY STATUS
podinfo oci://ghcr.io/stefanprodan/manifests/podinfo 5s True stored artifact with digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de'
podinfo oci://ghcr.io/stefanprodan/manifests/podinfo 5s True stored artifact with revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de'
```
3. Run `kubectl describe ocirepository podinfo` to see the [Artifact](#artifact)
@ -62,17 +62,17 @@ You can run this example by saving the manifest into `ocirepository.yaml`.
Checksum: d7e924b4882e55b97627355c7b3d2e711e9b54303afa2f50c25377f4df66a83b
Last Update Time: 2022-06-14T11:23:36Z
Path: ocirepository/default/podinfo/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de.tar.gz
Revision: 3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de
Revision: latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de
URL: http://source-controller.flux-system.svc.cluster.local./ocirepository/oci/podinfo/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de.tar.gz
Conditions:
Last Transition Time: 2022-06-14T11:23:36Z
Message: stored artifact for digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de'
Message: stored artifact for revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de'
Observed Generation: 1
Reason: Succeeded
Status: True
Type: Ready
Last Transition Time: 2022-06-14T11:23:36Z
Message: stored artifact for digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de'
Message: stored artifact for revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de'
Observed Generation: 1
Reason: Succeeded
Status: True
@ -82,7 +82,7 @@ You can run this example by saving the manifest into `ocirepository.yaml`.
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal NewArtifact 62s source-controller stored artifact with digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' from 'oci://ghcr.io/stefanprodan/manifests/podinfo'
Normal NewArtifact 62s source-controller stored artifact with revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de' from 'oci://ghcr.io/stefanprodan/manifests/podinfo'
```
## Writing an OCIRepository spec
@ -391,6 +391,7 @@ metadata:
spec:
layerSelector:
mediaType: "application/deployment.content.v1.tar+gzip"
operation: extract # can be 'extract' or 'copy', defaults to 'extract'
```
If the layer selector matches more than one layer, the first layer matching the specified media type will be used.
@ -398,6 +399,10 @@ Note that the selected OCI layer must be
[compressed](https://github.com/opencontainers/image-spec/blob/v1.0.2/layer.md#gzip-media-types)
in the `tar+gzip` format.
When `.spec.layerSelector.operation` is set to `copy`, instead of extracting the
compressed layer, the controller copies the tarball as-is to storage, thus
keeping the original content unaltered.
### Ignore
`.spec.ignore` is an optional field to specify rules in [the `.gitignore`
@ -673,8 +678,8 @@ lists
```console
LAST SEEN TYPE REASON OBJECT MESSAGE
2m14s Normal NewArtifact ocirepository/<repository-name> stored artifact for digest '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de'
36s Normal ArtifactUpToDate ocirepository/<repository-name> artifact up-to-date with remote digest: '3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de'
2m14s Normal NewArtifact ocirepository/<repository-name> stored artifact for revision 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de'
36s Normal ArtifactUpToDate ocirepository/<repository-name> artifact up-to-date with remote revision: 'latest/3b6cdcc7adcc9a84d3214ee1c029543789d90b5ae69debe9efa3f66e982875de'
94s Warning OCIOperationFailed ocirepository/<repository-name> failed to pull artifact from 'oci://ghcr.io/stefanprodan/manifests/podinfo': couldn't find tag "0.0.1"
```
@ -690,7 +695,7 @@ specific OCIRepository, e.g.
The OCIRepository reports the latest synchronized state from the OCI repository
as an Artifact object in the `.status.artifact` of the resource.
The `.status.artifact.revision` holds the SHA256 digest of the upstream OCI artifact.
The `.status.artifact.revision` holds the tag and SHA256 digest of the upstream OCI artifact.
The `.status.artifact.metadata` holds the upstream OCI artifact metadata such as the
[OpenContainers standard annotations](https://github.com/opencontainers/image-spec/blob/main/annotations.md).
@ -719,7 +724,7 @@ status:
org.opencontainers.image.revision: 6.1.8/b3b00fe35424a45d373bf4c7214178bc36fd7872
org.opencontainers.image.source: https://github.com/stefanprodan/podinfo.git
path: ocirepository/<namespace>/<repository-name>/<digest>.tar.gz
revision: <digest>
revision: <tag>/<digest>
url: http://source-controller.<namespace>.svc.cluster.local./ocirepository/<namespace>/<repository-name>/<digest>.tar.gz
```