Merge pull request #871 from fluxcd/oci-mediatype

[RFC-0003] Select layer by OCI media type

api/v1beta2/ocirepository_types.go

@@ -60,6 +60,11 @@ type OCIRepositorySpec struct {
 	// +optional
 	Reference *OCIRepositoryRef `json:"ref,omitempty"`
 
+	// LayerSelector specifies which layer should be extracted from the OCI artifact.
+	// When not specified, the first layer found in the artifact is selected.
+	// +optional
+	LayerSelector *OCILayerSelector `json:"layerSelector,omitempty"`
+
 	// The provider used for authentication, can be 'aws', 'azure', 'gcp' or 'generic'.
 	// When not specified, defaults to 'generic'.
 	// +kubebuilder:validation:Enum=generic;aws;azure;gcp
@@ -130,6 +135,15 @@ type OCIRepositoryRef struct {
 	Tag string `json:"tag,omitempty"`
 }
 
+// OCILayerSelector specifies which layer should be extracted from an OCI Artifact
+type OCILayerSelector struct {
+	// MediaType specifies the OCI media type of the layer
+	// which should be extracted from the OCI Artifact. The
+	// first layer matching this type is selected.
+	// +optional
+	MediaType string `json:"mediaType,omitempty"`
+}
+
 // OCIRepositoryVerification verifies the authenticity of an OCI Artifact
 type OCIRepositoryVerification struct {
 	// Provider specifies the technology used to sign the OCI Artifact.
@@ -192,6 +206,15 @@ func (in *OCIRepository) GetArtifact() *Artifact {
 	return in.Status.Artifact
 }
 
+// GetLayerMediaType returns the media type layer selector if found in spec.
+func (in *OCIRepository) GetLayerMediaType() string {
+	if in.Spec.LayerSelector == nil {
+		return ""
+	}
+
+	return in.Spec.LayerSelector.MediaType
+}
+
 // +genclient
 // +genclient:Namespaced
 // +kubebuilder:storageversion

api/v1beta2/zz_generated.deepcopy.go

@@ -622,6 +622,21 @@ func (in *LocalHelmChartSourceReference) DeepCopy() *LocalHelmChartSourceReferen
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *OCILayerSelector) DeepCopyInto(out *OCILayerSelector) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OCILayerSelector.
+func (in *OCILayerSelector) DeepCopy() *OCILayerSelector {
+	if in == nil {
+		return nil
+	}
+	out := new(OCILayerSelector)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *OCIRepository) DeepCopyInto(out *OCIRepository) {
 	*out = *in
@@ -704,6 +719,11 @@ func (in *OCIRepositorySpec) DeepCopyInto(out *OCIRepositorySpec) {
 		*out = new(OCIRepositoryRef)
 		**out = **in
 	}
+	if in.LayerSelector != nil {
+		in, out := &in.LayerSelector, &out.LayerSelector
+		*out = new(OCILayerSelector)
+		**out = **in
+	}
 	if in.SecretRef != nil {
 		in, out := &in.SecretRef, &out.SecretRef
 		*out = new(meta.LocalObjectReference)

config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml

@@ -75,6 +75,17 @@ spec:
               interval:
                 description: The interval at which to check for image updates.
                 type: string
+              layerSelector:
+                description: LayerSelector specifies which layer should be extracted
+                  from the OCI artifact. When not specified, the first layer found
+                  in the artifact is selected.
+                properties:
+                  mediaType:
+                    description: MediaType specifies the OCI media type of the layer
+                      which should be extracted from the OCI Artifact. The first layer
+                      matching this type is selected.
+                    type: string
+                type: object
               provider:
                 default: generic
                 description: The provider used for authentication, can be 'aws', 'azure',

controllers/ocirepository_controller.go

@@ -33,6 +33,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/authn/k8schain"
 	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/google/go-containerregistry/pkg/name"
+	gcrv1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -433,7 +434,40 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour
 			return sreconcile.ResultEmpty, e
 		}
 
-		blob, err := layers[0].Compressed()
+		var layer gcrv1.Layer
+
+		switch {
+		case obj.GetLayerMediaType() != "":
+			var found bool
+			for i, l := range layers {
+				md, err := l.MediaType()
+				if err != nil {
+					e := serror.NewGeneric(
+						fmt.Errorf("failed to determine the media type of layer[%v] from artifact: %w", i, err),
+						sourcev1.OCILayerOperationFailedReason,
+					)
+					conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
+					return sreconcile.ResultEmpty, e
+				}
+				if string(md) == obj.GetLayerMediaType() {
+					layer = layers[i]
+					found = true
+					break
+				}
+			}
+			if !found {
+				e := serror.NewGeneric(
+					fmt.Errorf("failed to find layer with media type '%s' in artifact", obj.GetLayerMediaType()),
+					sourcev1.OCILayerOperationFailedReason,
+				)
+				conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
+				return sreconcile.ResultEmpty, e
+			}
+		default:
+			layer = layers[0]
+		}
+
+		blob, err := layer.Compressed()
 		if err != nil {
 			e := serror.NewGeneric(
 				fmt.Errorf("failed to extract the first layer from artifact: %w", err),

controllers/ocirepository_controller_test.go

@@ -80,6 +80,7 @@ func TestOCIRepository_Reconcile(t *testing.T) {
 		tag            string
 		semver         string
 		digest         string
+		mediaType      string
 		assertArtifact []artifactFixture
 	}{
 		{
@@ -87,6 +88,7 @@ func TestOCIRepository_Reconcile(t *testing.T) {
 			url:       podinfoVersions["6.1.6"].url,
 			tag:       podinfoVersions["6.1.6"].tag,
 			digest:    podinfoVersions["6.1.6"].digest.Hex,
+			mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
 			assertArtifact: []artifactFixture{
 				{
 					expectedPath:     "kustomize/deployment.yaml",
@@ -142,7 +144,9 @@ func TestOCIRepository_Reconcile(t *testing.T) {
 			if tt.semver != "" {
 				obj.Spec.Reference.SemVer = tt.semver
 			}
-
+			if tt.mediaType != "" {
+				obj.Spec.LayerSelector = &sourcev1.OCILayerSelector{MediaType: tt.mediaType}
+			}
 			g.Expect(testEnv.Create(ctx, obj)).To(Succeed())
 
 			key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}
@@ -244,6 +248,109 @@ func TestOCIRepository_Reconcile(t *testing.T) {
 	}
 }
 
+func TestOCIRepository_Reconcile_MediaType(t *testing.T) {
+	g := NewWithT(t)
+
+	// Registry server with public images
+	tmpDir := t.TempDir()
+	regServer, err := setupRegistryServer(ctx, tmpDir, registryOptions{})
+	if err != nil {
+		g.Expect(err).ToNot(HaveOccurred())
+	}
+
+	podinfoVersions, err := pushMultiplePodinfoImages(regServer.registryHost, "6.1.4", "6.1.5", "6.1.6")
+
+	tests := []struct {
+		name      string
+		url       string
+		tag       string
+		mediaType string
+		wantErr   bool
+	}{
+		{
+			name: "Works with no media type",
+			url:  podinfoVersions["6.1.4"].url,
+			tag:  podinfoVersions["6.1.4"].tag,
+		},
+		{
+			name:      "Works with Flux CLI media type",
+			url:       podinfoVersions["6.1.5"].url,
+			tag:       podinfoVersions["6.1.5"].tag,
+			mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
+		},
+		{
+			name:      "Fails with unknown media type",
+			url:       podinfoVersions["6.1.6"].url,
+			tag:       podinfoVersions["6.1.6"].tag,
+			mediaType: "application/invalid.tar.gzip",
+			wantErr:   true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			g := NewWithT(t)
+
+			ns, err := testEnv.CreateNamespace(ctx, "ocirepository-mediatype-test")
+			g.Expect(err).ToNot(HaveOccurred())
+			defer func() { g.Expect(testEnv.Delete(ctx, ns)).To(Succeed()) }()
+
+			obj := &sourcev1.OCIRepository{
+				ObjectMeta: metav1.ObjectMeta{
+					GenerateName: "ocirepository-reconcile",
+					Namespace:    ns.Name,
+				},
+				Spec: sourcev1.OCIRepositorySpec{
+					URL:      tt.url,
+					Interval: metav1.Duration{Duration: 60 * time.Minute},
+					Reference: &sourcev1.OCIRepositoryRef{
+						Tag: tt.tag,
+					},
+					LayerSelector: &sourcev1.OCILayerSelector{
+						MediaType: tt.mediaType,
+					},
+				},
+			}
+
+			g.Expect(testEnv.Create(ctx, obj)).To(Succeed())
+
+			key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}
+
+			// Wait for the finalizer to be set
+			g.Eventually(func() bool {
+				if err := testEnv.Get(ctx, key, obj); err != nil {
+					return false
+				}
+				return len(obj.Finalizers) > 0
+			}, timeout).Should(BeTrue())
+
+			// Wait for the object to be reconciled
+			g.Eventually(func() bool {
+				if err := testEnv.Get(ctx, key, obj); err != nil {
+					return false
+				}
+				readyCondition := conditions.Get(obj, meta.ReadyCondition)
+				return readyCondition != nil
+			}, timeout).Should(BeTrue())
+
+			g.Expect(conditions.IsReady(obj)).To(BeIdenticalTo(!tt.wantErr))
+			if tt.wantErr {
+				g.Expect(conditions.Get(obj, meta.ReadyCondition).Message).Should(ContainSubstring("failed to find layer with media type"))
+			}
+
+			// Wait for the object to be deleted
+			g.Expect(testEnv.Delete(ctx, obj)).To(Succeed())
+			g.Eventually(func() bool {
+				if err := testEnv.Get(ctx, key, obj); err != nil {
+					return apierrors.IsNotFound(err)
+				}
+				return false
+			}, timeout).Should(BeTrue())
+		})
+	}
+}
+
 func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
 	type secretOptions struct {
 		username      string

docs/api/source.md

@@ -968,6 +968,21 @@ defaults to the latest tag.</p>
 </tr>
 <tr>
 <td>
+<code>layerSelector</code><br>
+<em>
+<a href="#source.toolkit.fluxcd.io/v1beta2.OCILayerSelector">
+OCILayerSelector
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>LayerSelector specifies which layer should be extracted from the OCI artifact.
+When not specified, the first layer found in the artifact is selected.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>provider</code><br>
 <em>
 string
@@ -2529,6 +2544,41 @@ string
 </table>
 </div>
 </div>
+<h3 id="source.toolkit.fluxcd.io/v1beta2.OCILayerSelector">OCILayerSelector
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#source.toolkit.fluxcd.io/v1beta2.OCIRepositorySpec">OCIRepositorySpec</a>)
+</p>
+<p>OCILayerSelector specifies which layer should be extracted from an OCI Artifact</p>
+<div class="md-typeset__scrollwrap">
+<div class="md-typeset__table">
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>mediaType</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>MediaType specifies the OCI media type of the layer
+which should be extracted from the OCI Artifact. The
+first layer matching this type is selected.</p>
+</td>
+</tr>
+</tbody>
+</table>
+</div>
+</div>
 <h3 id="source.toolkit.fluxcd.io/v1beta2.OCIRepositoryRef">OCIRepositoryRef
 </h3>
 <p>
@@ -2634,6 +2684,21 @@ defaults to the latest tag.</p>
 </tr>
 <tr>
 <td>
+<code>layerSelector</code><br>
+<em>
+<a href="#source.toolkit.fluxcd.io/v1beta2.OCILayerSelector">
+OCILayerSelector
+</a>
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>LayerSelector specifies which layer should be extracted from the OCI artifact.
+When not specified, the first layer found in the artifact is selected.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>provider</code><br>
 <em>
 string

docs/spec/v1beta2/ocirepositories.md

@@ -368,6 +368,30 @@ spec:
 
 This field takes precedence over all other fields.
 
+### Layer selector
+
+`spec.layerSelector` is an optional field to specify which layer should be extracted from the OCI Artifact.
+If not specified, the controller will extract the first layer found in the artifact.
+
+To extract a layer matching a specific
+[OCI media type](https://github.com/opencontainers/image-spec/blob/v1.0.2/media-types.md):
+
+```yaml
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: OCIRepository
+metadata:
+  name: <repository-name>
+spec:
+  layerSelector:
+    mediaType: "application/deployment.content.v1.tar+gzip"
+```
+
+If the layer selector matches more than one layer, the first layer matching the specified media type will be used.
+Note that the selected OCI layer must be
+[compressed](https://github.com/opencontainers/image-spec/blob/v1.0.2/layer.md#gzip-media-types)
+in the `tar+gzip` format.
+
 ### Ignore
 
 `.spec.ignore` is an optional field to specify rules in [the `.gitignore`