fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add the OCI metadata to the internal artifact 

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>
This commit is contained in:
Stefan Prodan 2022-07-12 18:21:08 +03:00
parent 5072091eb5
commit 05f9c0ee2b
No known key found for this signature in database
GPG Key ID: 3299AEB0E4085BAF
12 changed files with 112 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/v1beta2/artifact_types.go
View File

@ -54,6 +54,10 @@ type Artifact struct {
// Size is the number of bytes in the file.
// +optional
Size *int64 `json:"size,omitempty"`
// Metadata holds upstream information such as OCI annotations.
// +optional
Metadata map[string]string `json:"metadata,omitempty"`
}
// HasRevision returns if the given revision matches the current Revision of

6
api/v1beta2/ocirepository_types.go
View File

@ -17,9 +17,11 @@ limitations under the License.
package v1beta2
import (
"github.com/fluxcd/pkg/apis/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fluxcd/pkg/apis/meta"
)
const (

7
api/v1beta2/zz_generated.deepcopy.go
View File

@ -37,6 +37,13 @@ func (in *Artifact) DeepCopyInto(out *Artifact) {
*out = new(int64)
**out = **in
}
if in.Metadata != nil {
in, out := &in.Metadata, &out.Metadata
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Artifact.

5
config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml
View File

@ -384,6 +384,11 @@ spec:
the last update of the Artifact.
format: date-time
type: string
metadata:
additionalProperties:
type: string
description: Metadata holds upstream information such as OCI annotations.
type: object
path:
description: Path is the relative file path of the Artifact. It
can be used to locate the file in the root of the Artifact storage

11
config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml
View File

@ -559,6 +559,11 @@ spec:
the last update of the Artifact.
format: date-time
type: string
metadata:
additionalProperties:
type: string
description: Metadata holds upstream information such as OCI annotations.
type: object
path:
description: Path is the relative file path of the Artifact. It
can be used to locate the file in the root of the Artifact storage
@ -677,6 +682,12 @@ spec:
the last update of the Artifact.
format: date-time
type: string
metadata:
additionalProperties:
type: string
description: Metadata holds upstream information such as OCI
annotations.
type: object
path:
description: Path is the relative file path of the Artifact.
It can be used to locate the file in the root of the Artifact

5
config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml
View File

@ -432,6 +432,11 @@ spec:
the last update of the Artifact.
format: date-time
type: string
metadata:
additionalProperties:
type: string
description: Metadata holds upstream information such as OCI annotations.
type: object
path:
description: Path is the relative file path of the Artifact. It
can be used to locate the file in the root of the Artifact storage

5
config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml
View File

@ -362,6 +362,11 @@ spec:
the last update of the Artifact.
format: date-time
type: string
metadata:
additionalProperties:
type: string
description: Metadata holds upstream information such as OCI annotations.
type: object
path:
description: Path is the relative file path of the Artifact. It
can be used to locate the file in the root of the Artifact storage

5
config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml
View File

@ -142,6 +142,11 @@ spec:
the last update of the Artifact.
format: date-time
type: string
metadata:
additionalProperties:
type: string
description: Metadata holds upstream information such as OCI annotations.
type: object
path:
description: Path is the relative file path of the Artifact. It
can be used to locate the file in the root of the Artifact storage

45
controllers/ocirepository_controller.go
View File

@ -33,7 +33,6 @@ import (
"github.com/google/go-containerregistry/pkg/authn/k8schain"
"github.com/google/go-containerregistry/pkg/crane"
"github.com/google/go-containerregistry/pkg/name"
gcrv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
@ -110,7 +109,7 @@ var ociRepositoryFailConditions = []string{
// ociRepositoryReconcileFunc is the function type for all the v1beta2.OCIRepository
// (sub)reconcile functions. The type implementations are grouped and
// executed serially to perform the complete reconcile of the object.
type ociRepositoryReconcileFunc func(ctx context.Context, obj *sourcev1.OCIRepository, digest *gcrv1.Hash, dir string) (sreconcile.Result, error)
type ociRepositoryReconcileFunc func(ctx context.Context, obj *sourcev1.OCIRepository, metadata *sourcev1.Artifact, dir string) (sreconcile.Result, error)
// OCIRepositoryReconciler reconciles a v1beta2.OCIRepository object
type OCIRepositoryReconciler struct {
@ -261,16 +260,15 @@ func (r *OCIRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.O
}()
conditions.Delete(obj, sourcev1.StorageOperationFailedCondition)
hs := gcrv1.Hash{}
var (
res sreconcile.Result
resErr error
digest = hs.DeepCopy()
res sreconcile.Result
resErr error
metadata = sourcev1.Artifact{}
)
// Run the sub-reconcilers and build the result of reconciliation.
for _, rec := range reconcilers {
recResult, err := rec(ctx, obj, digest, tmpDir)
recResult, err := rec(ctx, obj, &metadata, tmpDir)
// Exit immediately on ResultRequeue.
if recResult == sreconcile.ResultRequeue {
return sreconcile.ResultRequeue, nil
@ -286,14 +284,14 @@ func (r *OCIRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.O
res = sreconcile.LowestRequeuingResult(res, recResult)
}
r.notify(ctx, oldObj, obj, digest, res, resErr)
r.notify(ctx, oldObj, obj, res, resErr)
return res, resErr
}
// reconcileSource fetches the upstream OCI artifact metadata and content.
// If this fails, it records v1beta2.FetchFailedCondition=True on the object and returns early.
func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sourcev1.OCIRepository, digest *gcrv1.Hash, dir string) (sreconcile.Result, error) {
func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sourcev1.OCIRepository, metadata *sourcev1.Artifact, dir string) (sreconcile.Result, error) {
ctxTimeout, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
defer cancel()
@ -352,9 +350,25 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour
}
// Set the internal revision to the remote digest hex
imgDigest.DeepCopyInto(digest)
revision := imgDigest.Hex
// Copy the OCI annotations to the internal artifact metadata
manifest, err := img.Manifest()
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to parse artifact manifest: %w", err),
sourcev1.OCIOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
m := &sourcev1.Artifact{
Revision: revision,
Metadata: manifest.Annotations,
}
m.DeepCopyInto(metadata)
// Mark observations about the revision on the object
defer func() {
if !obj.GetArtifact().HasRevision(revision) {
@ -606,7 +620,7 @@ func (r *OCIRepositoryReconciler) craneOptions(ctx context.Context,
// The hostname of any URL in the Status of the object are updated, to ensure
// they match the Storage server hostname of current runtime.
func (r *OCIRepositoryReconciler) reconcileStorage(ctx context.Context,
obj *sourcev1.OCIRepository, _ *gcrv1.Hash, _ string) (sreconcile.Result, error) {
obj *sourcev1.OCIRepository, _ *sourcev1.Artifact, _ string) (sreconcile.Result, error) {
// Garbage collect previous advertised artifact(s) from storage
_ = r.garbageCollect(ctx, obj)
@ -642,9 +656,9 @@ func (r *OCIRepositoryReconciler) reconcileStorage(ctx context.Context,
// On a successful archive, the Artifact in the Status of the object is set,
// and the symlink in the Storage is updated to its path.
func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context,
obj *sourcev1.OCIRepository, digest *gcrv1.Hash, dir string) (sreconcile.Result, error) {
obj *sourcev1.OCIRepository, metadata *sourcev1.Artifact, dir string) (sreconcile.Result, error) {
// Calculate revision
revision := digest.Hex
revision := metadata.Revision
// Create artifact
artifact := r.Storage.NewArtifactFor(obj.Kind, obj, revision, fmt.Sprintf("%s.tar.gz", revision))
@ -712,6 +726,7 @@ func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context,
// Record it on the object
obj.Status.Artifact = artifact.DeepCopy()
obj.Status.Artifact.Metadata = metadata.Metadata
// Update symlink on a "best effort" basis
url, err := r.Storage.Symlink(artifact, "latest.tar.gz")
@ -798,7 +813,7 @@ func (r *OCIRepositoryReconciler) eventLogf(ctx context.Context,
// notify emits notification related to the reconciliation.
func (r *OCIRepositoryReconciler) notify(ctx context.Context,
oldObj, newObj *sourcev1.OCIRepository, digest *gcrv1.Hash, res sreconcile.Result, resErr error) {
oldObj, newObj *sourcev1.OCIRepository, res sreconcile.Result, resErr error) {
// Notify successful reconciliation for new artifact and recovery from any
// failure.
if resErr == nil && res == sreconcile.ResultSuccess && newObj.Status.Artifact != nil {
@ -812,7 +827,7 @@ func (r *OCIRepositoryReconciler) notify(ctx context.Context,
oldChecksum = oldObj.GetArtifact().Checksum
}
message := fmt.Sprintf("stored artifact with digest '%s' from '%s'", digest.String(), newObj.Spec.URL)
message := fmt.Sprintf("stored artifact with digest '%s' from '%s'", newObj.Status.Artifact.Revision, newObj.Spec.URL)
// Notify on new artifact and failure recovery.
if oldChecksum != newObj.GetArtifact().Checksum {

33
controllers/ocirepository_controller_test.go
View File

@ -44,7 +44,8 @@ import (
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/crane"
"github.com/google/go-containerregistry/pkg/registry"
v1 "github.com/google/go-containerregistry/pkg/v1"
gcrv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/mutate"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@ -163,11 +164,13 @@ func TestOCIRepository_Reconcile(t *testing.T) {
obj.Generation == obj.Status.ObservedGeneration
}, timeout).Should(BeTrue())
t.Log(obj.Spec.Reference)
// Check if the revision matches the expected digest
g.Expect(obj.Status.Artifact.Revision).To(Equal(tt.digest))
// Check if the metadata matches the expected annotations
g.Expect(obj.Status.Artifact.Metadata["org.opencontainers.image.source"]).To(ContainSubstring("podinfo"))
g.Expect(obj.Status.Artifact.Metadata["org.opencontainers.image.revision"]).To(ContainSubstring(tt.tag))
// Check if the artifact storage path matches the expected file path
localPath := testStorage.LocalPath(*obj.Status.Artifact)
t.Logf("artifact local path: %s", localPath)
@ -252,6 +255,7 @@ func TestOCIRepository_SecretRef(t *testing.T) {
ociURL := fmt.Sprintf("oci://%s", repositoryURL)
// Push Test Image
image = setPodinfoImageAnnotations(image, "6.1.6")
err = crane.Push(image, repositoryURL, crane.WithAuth(&authn.Basic{
Username: testRegistryUsername,
Password: testRegistryPassword,
@ -265,7 +269,7 @@ func TestOCIRepository_SecretRef(t *testing.T) {
tests := []struct {
name string
url string
digest v1.Hash
digest gcrv1.Hash
includeSecretRef bool
includeServiceAccount bool
}{
@ -449,6 +453,7 @@ func TestOCIRepository_FailedAuth(t *testing.T) {
ociURL := fmt.Sprintf("oci://%s", repositoryURL)
// Push Test Image
image = setPodinfoImageAnnotations(image, "6.1.6")
err = crane.Push(image, repositoryURL, crane.WithAuth(&authn.Basic{
Username: testRegistryUsername,
Password: testRegistryPassword,
@ -462,7 +467,7 @@ func TestOCIRepository_FailedAuth(t *testing.T) {
tests := []struct {
name string
url string
digest v1.Hash
digest gcrv1.Hash
repoUsername string
repoPassword string
includeSecretRef bool
@ -644,7 +649,7 @@ func TestOCIRepository_CertSecret(t *testing.T) {
name string
url string
tag string
digest v1.Hash
digest gcrv1.Hash
certSecret *corev1.Secret
expectreadyconition bool
expectedstatusmessage string
@ -760,7 +765,7 @@ type artifactFixture struct {
type podinfoImage struct {
url string
tag string
digest v1.Hash
digest gcrv1.Hash
}
func createPodinfoImageFromTar(tarFileName, tag string, imageServer *httptest.Server) (*podinfoImage, error) {
@ -770,6 +775,8 @@ func createPodinfoImageFromTar(tarFileName, tag string, imageServer *httptest.Se
return nil, err
}
image = setPodinfoImageAnnotations(image, tag)
url, err := url.Parse(imageServer.URL)
if err != nil {
return nil, err
@ -784,7 +791,6 @@ func createPodinfoImageFromTar(tarFileName, tag string, imageServer *httptest.Se
// Push image
err = crane.Push(image, repositoryURL, crane.WithTransport(imageServer.Client().Transport))
if err != nil {
return nil, err
}
@ -802,8 +808,15 @@ func createPodinfoImageFromTar(tarFileName, tag string, imageServer *httptest.Se
}, nil
}
// These two taken verbatim from https://ericchiang.github.io/post/go-tls/
func setPodinfoImageAnnotations(img gcrv1.Image, tag string) gcrv1.Image {
metadata := map[string]string{
"org.opencontainers.image.source": "https://github.com/stefanprodan/podinfo",
"org.opencontainers.image.revision": fmt.Sprintf("%s/SHA", tag),
}
return mutate.Annotations(img, metadata).(gcrv1.Image)
}
// These two taken verbatim from https://ericchiang.github.io/post/go-tls/
func certTemplate() (*x509.Certificate, error) {
// generate a random serial number (a real cert authority would
// have some logic behind this)
@ -842,8 +855,6 @@ func createCert(template, parent *x509.Certificate, pub interface{}, parentPriv
return
}
// ----
func createTLSServer() (*httptest.Server, []byte, []byte, []byte, tls.Certificate, error) {
var clientTLSCert tls.Certificate
var rootCertPEM, clientCertPEM, clientKeyPEM []byte

2
controllers/suite_test.go
View File

@ -161,6 +161,8 @@ func setupRegistryServer(ctx context.Context) (*registryClientTestServer, error)
server.registryHost = fmt.Sprintf("localhost:%d", port)
config.HTTP.Addr = fmt.Sprintf("127.0.0.1:%d", port)
config.HTTP.DrainTimeout = time.Duration(10) * time.Second
config.Log.AccessLog.Disabled = true
config.Log.Level = "error"
config.Storage = map[string]configuration.Parameters{"inmemory": map[string]interface{}{}}
config.Auth = configuration.Auth{
"htpasswd": configuration.Parameters{

12
docs/api/source.md
View File

@ -1190,6 +1190,18 @@ int64
<p>Size is the number of bytes in the file.</p>
</td>
</tr>
<tr>
<td>
<code>metadata</code><br>
<em>
map[string]string
</em>
</td>
<td>
<em>(Optional)</em>
<p>Metadata holds upstream information such as OCI annotations.</p>
</td>
</tr>
</tbody>
</table>
</div>