fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #926 from fluxcd/status-observed-source-config 

Add explicit observed artifact content configurations in status
This commit is contained in:
Stefan Prodan 2022-10-11 13:46:05 +03:00 committed by GitHub
parent 70d9f126f9 a6d7948667
commit 5e83ecacec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 1023 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/v1beta2/bucket_types.go
View File

@ -128,6 +128,11 @@ type BucketStatus struct {
// +optional
Artifact *Artifact `json:"artifact,omitempty"`
// ObservedIgnore is the observed exclusion patterns used for constructing
// the source artifact.
// +optional
ObservedIgnore *string `json:"observedIgnore,omitempty"`
meta.ReconcileRequestStatus `json:",inline"`
}

18
api/v1beta2/gitrepository_types.go
View File

@ -224,9 +224,27 @@ type GitRepositoryStatus struct {
// be used to determine if the content of the included repository has
// changed.
// It has the format of `<algo>:<checksum>`, for example: `sha256:<checksum>`.
//
// Deprecated: Replaced with explicit fields for observed artifact content
// config in the status.
// +optional
ContentConfigChecksum string `json:"contentConfigChecksum,omitempty"`
// ObservedIgnore is the observed exclusion patterns used for constructing
// the source artifact.
// +optional
ObservedIgnore *string `json:"observedIgnore,omitempty"`
// ObservedRecurseSubmodules is the observed resource submodules
// configuration used to produce the current Artifact.
// +optional
ObservedRecurseSubmodules bool `json:"observedRecurseSubmodules,omitempty"`
// ObservedInclude is the observed list of GitRepository resources used to
// to produce the current Artifact.
// +optional
ObservedInclude []GitRepositoryInclude `json:"observedInclude,omitempty"`
meta.ReconcileRequestStatus `json:",inline"`
}

13
api/v1beta2/ocirepository_types.go
View File

@ -211,9 +211,22 @@ type OCIRepositoryStatus struct {
// be used to determine if the content configuration has changed and the
// artifact needs to be rebuilt.
// It has the format of `<algo>:<checksum>`, for example: `sha256:<checksum>`.
//
// Deprecated: Replaced with explicit fields for observed artifact content
// config in the status.
// +optional
ContentConfigChecksum string `json:"contentConfigChecksum,omitempty"`
// ObservedIgnore is the observed exclusion patterns used for constructing
// the source artifact.
// +optional
ObservedIgnore *string `json:"observedIgnore,omitempty"`
// ObservedLayerSelector is the observed layer selector used for constructing
// the source artifact.
// +optional
ObservedLayerSelector *OCILayerSelector `json:"observedLayerSelector,omitempty"`
meta.ReconcileRequestStatus `json:",inline"`
}

25
api/v1beta2/zz_generated.deepcopy.go
View File

@ -166,6 +166,11 @@ func (in *BucketStatus) DeepCopyInto(out *BucketStatus) {
*out = new(Artifact)
(*in).DeepCopyInto(*out)
}
if in.ObservedIgnore != nil {
in, out := &in.ObservedIgnore, &out.ObservedIgnore
*out = new(string)
**out = **in
}
out.ReconcileRequestStatus = in.ReconcileRequestStatus
}
@ -346,6 +351,16 @@ func (in *GitRepositoryStatus) DeepCopyInto(out *GitRepositoryStatus) {
}
}
}
if in.ObservedIgnore != nil {
in, out := &in.ObservedIgnore, &out.ObservedIgnore
*out = new(string)
**out = **in
}
if in.ObservedInclude != nil {
in, out := &in.ObservedInclude, &out.ObservedInclude
*out = make([]GitRepositoryInclude, len(*in))
copy(*out, *in)
}
out.ReconcileRequestStatus = in.ReconcileRequestStatus
}
@ -777,6 +792,16 @@ func (in *OCIRepositoryStatus) DeepCopyInto(out *OCIRepositoryStatus) {
*out = new(Artifact)
(*in).DeepCopyInto(*out)
}
if in.ObservedIgnore != nil {
in, out := &in.ObservedIgnore, &out.ObservedIgnore
*out = new(string)
**out = **in
}
if in.ObservedLayerSelector != nil {
in, out := &in.ObservedLayerSelector, &out.ObservedLayerSelector
*out = new(OCILayerSelector)
**out = **in
}
out.ReconcileRequestStatus = in.ReconcileRequestStatus
}

4
config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml
View File

@ -492,6 +492,10 @@ spec:
the Bucket object.
format: int64
type: integer
observedIgnore:
description: ObservedIgnore is the observed exclusion patterns used
for constructing the source artifact.
type: string
url:
description: URL is the dynamic fetch link for the latest Artifact.
It is provided on a "best effort" basis, and using the precise BucketStatus.Artifact

43
config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml
View File

@ -658,13 +658,14 @@ spec:
type: object
type: array
contentConfigChecksum:
description: 'ContentConfigChecksum is a checksum of all the configurations
description: "ContentConfigChecksum is a checksum of all the configurations
related to the content of the source artifact: - .spec.ignore -
.spec.recurseSubmodules - .spec.included and the checksum of the
included artifacts observed in .status.observedGeneration version
of the object. This can be used to determine if the content of the
included repository has changed. It has the format of `<algo>:<checksum>`,
for example: `sha256:<checksum>`.'
for example: `sha256:<checksum>`. \n Deprecated: Replaced with explicit
fields for observed artifact content config in the status."
type: string
includedArtifacts:
description: IncludedArtifacts contains a list of the last successfully
@ -723,6 +724,44 @@ spec:
the GitRepository object.
format: int64
type: integer
observedIgnore:
description: ObservedIgnore is the observed exclusion patterns used
for constructing the source artifact.
type: string
observedInclude:
description: ObservedInclude is the observed list of GitRepository
resources used to to produce the current Artifact.
items:
description: GitRepositoryInclude specifies a local reference to
a GitRepository which Artifact (sub-)contents must be included,
and where they should be placed.
properties:
fromPath:
description: FromPath specifies the path to copy contents from,
defaults to the root of the Artifact.
type: string
repository:
description: GitRepositoryRef specifies the GitRepository which
Artifact contents must be included.
properties:
name:
description: Name of the referent.
type: string
required:
- name
type: object
toPath:
description: ToPath specifies the path to copy contents to,
defaults to the name of the GitRepositoryRef.
type: string
required:
- repository
type: object
type: array
observedRecurseSubmodules:
description: ObservedRecurseSubmodules is the observed resource submodules
configuration used to produce the current Artifact.
type: boolean
url:
description: URL is the dynamic fetch link for the latest Artifact.
It is provided on a "best effort" basis, and using the precise GitRepositoryStatus.Artifact

29
config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml
View File

@ -301,12 +301,14 @@ spec:
type: object
type: array
contentConfigChecksum:
description: 'ContentConfigChecksum is a checksum of all the configurations
description: "ContentConfigChecksum is a checksum of all the configurations
related to the content of the source artifact: - .spec.ignore -
.spec.layerSelector observed in .status.observedGeneration version
of the object. This can be used to determine if the content configuration
has changed and the artifact needs to be rebuilt. It has the format
of `<algo>:<checksum>`, for example: `sha256:<checksum>`.'
of `<algo>:<checksum>`, for example: `sha256:<checksum>`. \n Deprecated:
Replaced with explicit fields for observed artifact content config
in the status."
type: string
lastHandledReconcileAt:
description: LastHandledReconcileAt holds the value of the most recent
@ -317,6 +319,29 @@ spec:
description: ObservedGeneration is the last observed generation.
format: int64
type: integer
observedIgnore:
description: ObservedIgnore is the observed exclusion patterns used
for constructing the source artifact.
type: string
observedLayerSelector:
description: ObservedLayerSelector is the observed layer selector
used for constructing the source artifact.
properties:
mediaType:
description: MediaType specifies the OCI media type of the layer
which should be extracted from the OCI Artifact. The first layer
matching this type is selected.
type: string
operation:
description: Operation specifies how the selected layer should
be processed. By default, the layer compressed content is extracted
to storage. When the operation is set to 'copy', the layer compressed
content is persisted to storage as it is.
enum:
- extract
- copy
type: string
type: object
url:
description: URL is the download link for the artifact output of the
last OCI Repository sync.

1
controllers/bucket_controller.go
View File

@ -628,6 +628,7 @@ func (r *BucketReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.
// Record it on the object
obj.Status.Artifact = artifact.DeepCopy()
obj.Status.ObservedIgnore = obj.Spec.Ignore
// Update symlink on a "best effort" basis
url, err := r.Storage.Symlink(artifact, "latest.tar.gz")

125
controllers/gitrepository_controller.go
View File

@ -18,12 +18,10 @@ package controllers
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@ -33,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
kuberecorder "k8s.io/client-go/tools/record"
"k8s.io/utils/pointer"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
@ -507,8 +506,8 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
// If it's a partial commit obtained from an existing artifact, check if the
// reconciliation can be skipped if other configurations have not changed.
if !git.IsConcreteCommit(*commit) {
// Calculate content configuration checksum.
if r.calculateContentConfigChecksum(obj, includes) == obj.Status.ContentConfigChecksum {
// Check if the content config contributing to the artifact has changed.
if !gitContentConfigChanged(obj, includes) {
ge := serror.NewGeneric(
fmt.Errorf("no changes since last reconcilation: observed revision '%s'",
commit.String()), sourcev1.GitOperationSucceedReason,
@ -559,27 +558,24 @@ func (r *GitRepositoryReconciler) reconcileSource(ctx context.Context,
//
// The inspection of the given data to the object is differed, ensuring any
// stale observations like v1beta2.ArtifactOutdatedCondition are removed.
// If the given Artifact and/or artifactSet (includes) and the content config
// checksum do not differ from the object's current, it returns early.
// If the given Artifact and/or artifactSet (includes) and observed artifact
// content config do not differ from the object's current, it returns early.
// Source ignore patterns are loaded, and the given directory is archived while
// taking these patterns into account.
// On a successful archive, the Artifact, Includes and new content config
// checksum in the Status of the object are set, and the symlink in the Storage
// is updated to its path.
// On a successful archive, the Artifact, Includes, observed ignore, recurse
// submodules and observed include in the Status of the object are set, and the
// symlink in the Storage is updated to its path.
func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context,
obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) {
// Create potential new artifact with current available metadata
artifact := r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), commit.String(), fmt.Sprintf("%s.tar.gz", commit.Hash.String()))
// Calculate the content config checksum.
ccc := r.calculateContentConfigChecksum(obj, includes)
// Set the ArtifactInStorageCondition if there's no drift.
defer func() {
if obj.GetArtifact().HasRevision(artifact.Revision) &&
!includes.Diff(obj.Status.IncludedArtifacts) &&
obj.Status.ContentConfigChecksum == ccc {
!gitContentConfigChanged(obj, includes) {
conditions.Delete(obj, sourcev1.ArtifactOutdatedCondition)
conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason,
"stored artifact for revision '%s'", artifact.Revision)
@ -589,7 +585,7 @@ func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context,
// The artifact is up-to-date
if obj.GetArtifact().HasRevision(artifact.Revision) &&
!includes.Diff(obj.Status.IncludedArtifacts) &&
obj.Status.ContentConfigChecksum == ccc {
!gitContentConfigChanged(obj, includes) {
r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason, "artifact up-to-date with remote revision: '%s'", artifact.Revision)
return sreconcile.ResultSuccess, nil
}
@ -652,10 +648,13 @@ func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context,
return sreconcile.ResultEmpty, e
}
// Record it on the object
// Record the observations on the object.
obj.Status.Artifact = artifact.DeepCopy()
obj.Status.IncludedArtifacts = *includes
obj.Status.ContentConfigChecksum = ccc
obj.Status.ContentConfigChecksum = "" // To be removed in the next API version.
obj.Status.ObservedIgnore = obj.Spec.Ignore
obj.Status.ObservedRecurseSubmodules = obj.Spec.RecurseSubmodules
obj.Status.ObservedInclude = obj.Spec.Include
// Update symlink on a "best effort" basis
url, err := r.Storage.Symlink(artifact, "latest.tar.gz")
@ -825,39 +824,6 @@ func (r *GitRepositoryReconciler) fetchIncludes(ctx context.Context, obj *source
return &artifacts, nil
}
// calculateContentConfigChecksum calculates a checksum of all the
// configurations that result in a change in the source artifact. It can be used
// to decide if further reconciliation is needed when an artifact already exists
// for a set of configurations.
func (r *GitRepositoryReconciler) calculateContentConfigChecksum(obj *sourcev1.GitRepository, includes *artifactSet) string {
c := []byte{}
// Consider the ignore rules and recurse submodules.
if obj.Spec.Ignore != nil {
c = append(c, []byte(*obj.Spec.Ignore)...)
}
c = append(c, []byte(strconv.FormatBool(obj.Spec.RecurseSubmodules))...)
// Consider the included repository attributes.
for _, incl := range obj.Spec.Include {
c = append(c, []byte(incl.GitRepositoryRef.Name+incl.FromPath+incl.ToPath)...)
}
// Consider the checksum and revision of all the included remote artifact.
// This ensures that if the included repos get updated, this checksum changes.
// NOTE: The content of an artifact may change at the same revision if the
// ignore rules change. Hence, consider both checksum and revision to
// capture changes in artifact checksum as well.
// TODO: Fix artifactSet.Diff() to consider checksum as well.
if includes != nil {
for _, incl := range *includes {
c = append(c, []byte(incl.Checksum)...)
c = append(c, []byte(incl.Revision)...)
}
}
return fmt.Sprintf("sha256:%x", sha256.Sum256(c))
}
// verifyCommitSignature verifies the signature of the given Git commit, if a
// verification mode is specified on the object.
// If the signature can not be verified or the verification fails, it records
@ -978,3 +944,64 @@ func (r *GitRepositoryReconciler) eventLogf(ctx context.Context, obj runtime.Obj
}
r.Eventf(obj, eventType, reason, msg)
}
// gitContentConfigChanged evaluates the current spec with the observations of
// the artifact in the status to determine if artifact content configuration has
// changed and requires rebuilding the artifact.
func gitContentConfigChanged(obj *sourcev1.GitRepository, includes *artifactSet) bool {
if !pointer.StringEqual(obj.Spec.Ignore, obj.Status.ObservedIgnore) {
return true
}
if obj.Spec.RecurseSubmodules != obj.Status.ObservedRecurseSubmodules {
return true
}
if len(obj.Spec.Include) != len(obj.Status.ObservedInclude) {
return true
}
// Convert artifactSet to index addressable artifacts and ensure that it and
// the included artifacts include all the include from the spec.
artifacts := []*sourcev1.Artifact(*includes)
if len(obj.Spec.Include) != len(artifacts) {
return true
}
if len(obj.Spec.Include) != len(obj.Status.IncludedArtifacts) {
return true
}
// The order of spec.include, status.IncludeArtifacts and
// status.observedInclude are the same. Compare the values by index.
for index, incl := range obj.Spec.Include {
observedIncl := obj.Status.ObservedInclude[index]
observedInclArtifact := obj.Status.IncludedArtifacts[index]
currentIncl := artifacts[index]
// Check if the include are the same in spec and status.
if !gitRepositoryIncludeEqual(incl, observedIncl) {
return true
}
// Check if the included repositories are still the same.
if observedInclArtifact.Revision != currentIncl.Revision {
return true
}
if observedInclArtifact.Checksum != currentIncl.Checksum {
return true
}
}
return false
}
// Returns true if both GitRepositoryIncludes are equal.
func gitRepositoryIncludeEqual(a, b sourcev1.GitRepositoryInclude) bool {
if a.GitRepositoryRef != b.GitRepositoryRef {
return false
}
if a.FromPath != b.FromPath {
return false
}
if a.ToPath != b.ToPath {
return false
}
return true
}

430
controllers/gitrepository_controller_test.go
View File

@ -143,7 +143,6 @@ Oomb3gD/TRf/nAdVED+k81GdLzciYdUGtI71/qI47G0nMBluLRE=
=/4e+
-----END PGP PUBLIC KEY BLOCK-----
`
emptyContentConfigChecksum = "sha256:fcbcf165908dd18a9e49f7ff27810176db8e9f63b4352213741664245224f8aa"
)
var (
@ -685,8 +684,6 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
Revision: "staging/" + latestRev,
Path: randStringRunes(10),
},
// Checksum with all the relevant fields unset.
ContentConfigChecksum: emptyContentConfigChecksum,
}
conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "foo")
},
@ -709,8 +706,6 @@ func TestGitRepositoryReconciler_reconcileSource_checkoutStrategy(t *testing.T)
Revision: "staging/" + latestRev,
Path: randStringRunes(10),
},
// Checksum with all the relevant fields unset.
ContentConfigChecksum: emptyContentConfigChecksum,
}
conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "foo")
},
@ -835,6 +830,9 @@ func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) {
includes: artifactSet{&sourcev1.Artifact{Revision: "main/revision"}},
beforeFunc: func(obj *sourcev1.GitRepository) {
obj.Spec.Interval = metav1.Duration{Duration: interval}
obj.Spec.Include = []sourcev1.GitRepositoryInclude{
{GitRepositoryRef: meta.LocalObjectReference{Name: "foo"}},
}
},
afterFunc: func(t *WithT, obj *sourcev1.GitRepository) {
t.Expect(obj.GetArtifact()).ToNot(BeNil())
@ -850,12 +848,15 @@ func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) {
{
name: "Up-to-date artifact should not update status",
dir: "testdata/git/repository",
includes: artifactSet{&sourcev1.Artifact{Revision: "main/revision"}},
includes: artifactSet{&sourcev1.Artifact{Revision: "main/revision", Checksum: "some-checksum"}},
beforeFunc: func(obj *sourcev1.GitRepository) {
obj.Spec.Interval = metav1.Duration{Duration: interval}
obj.Spec.Include = []sourcev1.GitRepositoryInclude{
{GitRepositoryRef: meta.LocalObjectReference{Name: "foo"}},
}
obj.Status.Artifact = &sourcev1.Artifact{Revision: "main/revision"}
obj.Status.IncludedArtifacts = []*sourcev1.Artifact{{Revision: "main/revision", Checksum: "some-checksum"}}
obj.Status.ContentConfigChecksum = "sha256:f825d11a1c5987e033d2cb36449a3b0435a6abc9b2bfdbcdcc7c49bf40e9285d"
obj.Status.ObservedInclude = obj.Spec.Include
},
afterFunc: func(t *WithT, obj *sourcev1.GitRepository) {
t.Expect(obj.Status.URL).To(BeEmpty())
@ -2145,53 +2146,6 @@ func TestGitRepositoryReconciler_fetchIncludes(t *testing.T) {
}
}
func TestGitRepositoryReconciler_calculateContentConfigChecksum(t *testing.T) {
g := NewWithT(t)
obj := &sourcev1.GitRepository{}
r := &GitRepositoryReconciler{}
emptyChecksum := r.calculateContentConfigChecksum(obj, nil)
g.Expect(emptyChecksum).To(Equal(emptyContentConfigChecksum))
// Ignore modified.
obj.Spec.Ignore = pointer.String("some-rule")
ignoreModChecksum := r.calculateContentConfigChecksum(obj, nil)
g.Expect(emptyChecksum).ToNot(Equal(ignoreModChecksum))
// Recurse submodules modified.
obj.Spec.RecurseSubmodules = true
submodModChecksum := r.calculateContentConfigChecksum(obj, nil)
g.Expect(ignoreModChecksum).ToNot(Equal(submodModChecksum))
// Include modified.
obj.Spec.Include = []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "aaa",
ToPath: "bbb",
},
}
artifacts := &artifactSet{
&sourcev1.Artifact{Revision: "some-revision-1", Checksum: "some-checksum-1"},
}
includeModChecksum := r.calculateContentConfigChecksum(obj, artifacts)
g.Expect(submodModChecksum).ToNot(Equal(includeModChecksum))
// Artifact modified revision.
artifacts = &artifactSet{
&sourcev1.Artifact{Revision: "some-revision-2", Checksum: "some-checksum-1"},
}
artifactModChecksum := r.calculateContentConfigChecksum(obj, artifacts)
g.Expect(includeModChecksum).ToNot(Equal(artifactModChecksum))
// Artifact modified checksum.
artifacts = &artifactSet{
&sourcev1.Artifact{Revision: "some-revision-2", Checksum: "some-checksum-2"},
}
artifactCsumModChecksum := r.calculateContentConfigChecksum(obj, artifacts)
g.Expect(artifactModChecksum).ToNot(Equal(artifactCsumModChecksum))
}
func resetChmod(path string, dirMode os.FileMode, fileMode os.FileMode) error {
err := filepath.Walk(path,
func(path string, info os.FileInfo, err error) error {
@ -2212,3 +2166,371 @@ func resetChmod(path string, dirMode os.FileMode, fileMode os.FileMode) error {
return nil
}
func TestGitRepositoryIncludeEqual(t *testing.T) {
tests := []struct {
name string
a sourcev1.GitRepositoryInclude
b sourcev1.GitRepositoryInclude
want bool
}{
{
name: "empty",
want: true,
},
{
name: "different refs",
a: sourcev1.GitRepositoryInclude{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
},
b: sourcev1.GitRepositoryInclude{
GitRepositoryRef: meta.LocalObjectReference{Name: "bar"},
},
want: false,
},
{
name: "same refs",
a: sourcev1.GitRepositoryInclude{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
},
b: sourcev1.GitRepositoryInclude{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
},
want: true,
},
{
name: "different from paths",
a: sourcev1.GitRepositoryInclude{FromPath: "foo"},
b: sourcev1.GitRepositoryInclude{FromPath: "bar"},
want: false,
},
{
name: "same from paths",
a: sourcev1.GitRepositoryInclude{FromPath: "foo"},
b: sourcev1.GitRepositoryInclude{FromPath: "foo"},
want: true,
},
{
name: "different to paths",
a: sourcev1.GitRepositoryInclude{ToPath: "foo"},
b: sourcev1.GitRepositoryInclude{ToPath: "bar"},
want: false,
},
{
name: "same to paths",
a: sourcev1.GitRepositoryInclude{ToPath: "foo"},
b: sourcev1.GitRepositoryInclude{ToPath: "foo"},
want: true,
},
{
name: "same all",
a: sourcev1.GitRepositoryInclude{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo-ref"},
FromPath: "foo-path",
ToPath: "bar-path",
},
b: sourcev1.GitRepositoryInclude{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo-ref"},
FromPath: "foo-path",
ToPath: "bar-path",
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
g.Expect(gitRepositoryIncludeEqual(tt.a, tt.b)).To(Equal(tt.want))
})
}
}
func TestGitContentConfigChanged(t *testing.T) {
tests := []struct {
name string
obj sourcev1.GitRepository
artifacts []*sourcev1.Artifact
want bool
}{
{
name: "no content config",
want: false,
},
{
name: "unobserved ignore",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{Ignore: pointer.String("foo")},
},
want: true,
},
{
name: "observed ignore",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{Ignore: pointer.String("foo")},
Status: sourcev1.GitRepositoryStatus{ObservedIgnore: pointer.String("foo")},
},
want: false,
},
{
name: "unobserved recurse submodules",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{RecurseSubmodules: true},
},
want: true,
},
{
name: "observed recurse submodules",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{RecurseSubmodules: true},
Status: sourcev1.GitRepositoryStatus{ObservedRecurseSubmodules: true},
},
want: false,
},
{
name: "unobserved include",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{
Include: []sourcev1.GitRepositoryInclude{
{GitRepositoryRef: meta.LocalObjectReference{Name: "foo"}, FromPath: "bar", ToPath: "baz"},
},
},
},
want: true,
},
{
name: "observed include",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{
Include: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
},
},
Status: sourcev1.GitRepositoryStatus{
ObservedInclude: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
},
IncludedArtifacts: []*sourcev1.Artifact{{Revision: "aaa", Checksum: "bbb"}},
},
},
artifacts: []*sourcev1.Artifact{
{Revision: "aaa", Checksum: "bbb"},
},
want: false,
},
{
name: "observed include but different artifact revision",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{
Include: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
},
},
Status: sourcev1.GitRepositoryStatus{
ObservedInclude: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
},
IncludedArtifacts: []*sourcev1.Artifact{{Revision: "aaa", Checksum: "bbb"}},
},
},
artifacts: []*sourcev1.Artifact{
{Revision: "ccc", Checksum: "bbb"},
},
want: true,
},
{
name: "observed include but different artifact checksum",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{
Include: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
},
},
Status: sourcev1.GitRepositoryStatus{
ObservedInclude: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
},
IncludedArtifacts: []*sourcev1.Artifact{{Revision: "aaa", Checksum: "bbb"}},
},
},
artifacts: []*sourcev1.Artifact{
{Revision: "aaa", Checksum: "ddd"},
},
want: true,
},
{
name: "observed include but updated spec",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{
Include: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo2"},
FromPath: "bar",
ToPath: "baz",
},
},
},
Status: sourcev1.GitRepositoryStatus{
ObservedInclude: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
},
IncludedArtifacts: []*sourcev1.Artifact{{Revision: "aaa", Checksum: "bbb"}},
},
},
artifacts: []*sourcev1.Artifact{
{Revision: "aaa", Checksum: "bbb"},
},
want: true,
},
{
name: "different number of include and observed include",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{
Include: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo2"},
FromPath: "bar",
ToPath: "baz",
},
},
},
Status: sourcev1.GitRepositoryStatus{
IncludedArtifacts: []*sourcev1.Artifact{
{Revision: "aaa", Checksum: "bbb"},
{Revision: "ccc", Checksum: "ccc"},
},
},
},
artifacts: []*sourcev1.Artifact{
{Revision: "aaa", Checksum: "bbb"},
{Revision: "ccc", Checksum: "ddd"},
},
want: true,
},
{
name: "different number of include and artifactset",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{
Include: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo2"},
FromPath: "bar",
ToPath: "baz",
},
},
},
Status: sourcev1.GitRepositoryStatus{
ObservedInclude: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo2"},
FromPath: "bar",
ToPath: "baz",
},
},
IncludedArtifacts: []*sourcev1.Artifact{
{Revision: "aaa", Checksum: "bbb"},
{Revision: "ccc", Checksum: "ccc"},
},
},
},
artifacts: []*sourcev1.Artifact{
{Revision: "aaa", Checksum: "bbb"},
},
want: true,
},
{
name: "different number of include and included artifacts",
obj: sourcev1.GitRepository{
Spec: sourcev1.GitRepositorySpec{
Include: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo2"},
FromPath: "bar",
ToPath: "baz",
},
},
},
Status: sourcev1.GitRepositoryStatus{
ObservedInclude: []sourcev1.GitRepositoryInclude{
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo"},
FromPath: "bar",
ToPath: "baz",
},
{
GitRepositoryRef: meta.LocalObjectReference{Name: "foo2"},
FromPath: "bar",
ToPath: "baz",
},
},
IncludedArtifacts: []*sourcev1.Artifact{
{Revision: "aaa", Checksum: "bbb"},
},
},
},
artifacts: []*sourcev1.Artifact{
{Revision: "aaa", Checksum: "bbb"},
{Revision: "ccc", Checksum: "ccc"},
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
includes := artifactSet(tt.artifacts)
g.Expect(gitContentConfigChanged(&tt.obj, &includes)).To(Equal(tt.want))
})
}
}

68
controllers/ocirepository_controller.go
View File

@ -18,7 +18,6 @@ package controllers
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"errors"
@ -44,6 +43,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
kuberecorder "k8s.io/client-go/tools/record"
"k8s.io/utils/pointer"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
@ -427,10 +427,9 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour
conditions.MarkTrue(obj, sourcev1.SourceVerifiedCondition, meta.SucceededReason, "verified signature of revision %s", revision)
}
// Skip pulling if the artifact revision and the content config checksum has
// Skip pulling if the artifact revision and the source configuration has
// not changed.
if obj.GetArtifact().HasRevision(revision) &&
r.calculateContentConfigChecksum(obj) == obj.Status.ContentConfigChecksum {
if obj.GetArtifact().HasRevision(revision) && !ociContentConfigChanged(obj) {
conditions.Delete(obj, sourcev1.FetchFailedCondition)
return sreconcile.ResultSuccess, nil
}
@ -918,13 +917,9 @@ func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *so
artifact := r.Storage.NewArtifactFor(obj.Kind, obj, revision,
fmt.Sprintf("%s.tar.gz", r.digestFromRevision(revision)))
// Calculate the content config checksum.
ccc := r.calculateContentConfigChecksum(obj)
// Set the ArtifactInStorageCondition if there's no drift.
defer func() {
if obj.GetArtifact().HasRevision(artifact.Revision) &&
obj.Status.ContentConfigChecksum == ccc {
if obj.GetArtifact().HasRevision(artifact.Revision) && !ociContentConfigChanged(obj) {
conditions.Delete(obj, sourcev1.ArtifactOutdatedCondition)
conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason,
"stored artifact for digest '%s'", artifact.Revision)
@ -932,8 +927,7 @@ func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *so
}()
// The artifact is up-to-date
if obj.GetArtifact().HasRevision(artifact.Revision) &&
obj.Status.ContentConfigChecksum == ccc {
if obj.GetArtifact().HasRevision(artifact.Revision) && !ociContentConfigChanged(obj) {
r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason,
"artifact up-to-date with remote revision: '%s'", artifact.Revision)
return sreconcile.ResultSuccess, nil
@ -1008,10 +1002,12 @@ func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context, obj *so
}
}
// Record it on the object
// Record the observations on the object.
obj.Status.Artifact = artifact.DeepCopy()
obj.Status.Artifact.Metadata = metadata.Metadata
obj.Status.ContentConfigChecksum = ccc
obj.Status.ContentConfigChecksum = "" // To be removed in the next API version.
obj.Status.ObservedIgnore = obj.Spec.Ignore
obj.Status.ObservedLayerSelector = obj.Spec.LayerSelector
// Update symlink on a "best effort" basis
url, err := r.Storage.Symlink(artifact, "latest.tar.gz")
@ -1141,24 +1137,6 @@ func (r *OCIRepositoryReconciler) notify(ctx context.Context, oldObj, newObj *so
}
}
// calculateContentConfigChecksum calculates a checksum of all the
// configurations that result in a change in the source artifact. It can be used
// to decide if further reconciliation is needed when an artifact already exists
// for a set of configurations.
func (r *OCIRepositoryReconciler) calculateContentConfigChecksum(obj *sourcev1.OCIRepository) string {
c := []byte{}
// Consider the ignore rules.
if obj.Spec.Ignore != nil {
c = append(c, []byte(*obj.Spec.Ignore)...)
}
// Consider the layer selector.
if obj.Spec.LayerSelector != nil {
c = append(c, []byte(obj.GetLayerMediaType()+obj.GetLayerOperation())...)
}
return fmt.Sprintf("sha256:%x", sha256.Sum256(c))
}
// craneOptions sets the auth headers, timeout and user agent
// for all operations against remote container registries.
func craneOptions(ctx context.Context, insecure bool) []crane.Option {
@ -1208,3 +1186,31 @@ type remoteOptions struct {
craneOpts []crane.Option
verifyOpts []remote.Option
}
// ociContentConfigChanged evaluates the current spec with the observations
// of the artifact in the status to determine if artifact content configuration
// has changed and requires rebuilding the artifact.
func ociContentConfigChanged(obj *sourcev1.OCIRepository) bool {
if !pointer.StringEqual(obj.Spec.Ignore, obj.Status.ObservedIgnore) {
return true
}
if !layerSelectorEqual(obj.Spec.LayerSelector, obj.Status.ObservedLayerSelector) {
return true
}
return false
}
// Returns true if both arguments are nil or both arguments
// dereference to the same value.
// Based on k8s.io/utils/pointer/pointer.go pointer value equality.
func layerSelectorEqual(a, b *sourcev1.OCILayerSelector) bool {
if (a == nil) != (b == nil) {
return false
}
if a == nil {
return true
}
return *a == *b
}

208
controllers/ocirepository_controller_test.go
View File

@ -69,8 +69,6 @@ import (
"github.com/fluxcd/source-controller/pkg/git"
)
const ociRepoEmptyContentConfigChecksum = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
func TestOCIRepository_Reconcile(t *testing.T) {
g := NewWithT(t)
@ -1290,21 +1288,48 @@ func TestOCIRepository_reconcileSource_noop(t *testing.T) {
},
},
{
name: "noop - artifact revisions and ccc match",
name: "noop - artifact revisions match",
beforeFunc: func(obj *sourcev1.OCIRepository) {
obj.Status.Artifact = &sourcev1.Artifact{
Revision: testRevision,
}
obj.Status.ContentConfigChecksum = ociRepoEmptyContentConfigChecksum
},
afterFunc: func(g *WithT, artifact *sourcev1.Artifact) {
g.Expect(artifact.Metadata).To(BeEmpty())
},
},
{
name: "full reconcile - same rev, different ccc",
name: "full reconcile - same rev, unobserved ignore",
beforeFunc: func(obj *sourcev1.OCIRepository) {
obj.Status.ContentConfigChecksum = "some-checksum"
obj.Status.ObservedIgnore = pointer.String("aaa")
obj.Status.Artifact = &sourcev1.Artifact{
Revision: testRevision,
}
},
afterFunc: func(g *WithT, artifact *sourcev1.Artifact) {
g.Expect(artifact.Metadata).ToNot(BeEmpty())
},
},
{
name: "noop - same rev, observed ignore",
beforeFunc: func(obj *sourcev1.OCIRepository) {
obj.Spec.Ignore = pointer.String("aaa")
obj.Status.ObservedIgnore = pointer.String("aaa")
obj.Status.Artifact = &sourcev1.Artifact{
Revision: testRevision,
}
},
afterFunc: func(g *WithT, artifact *sourcev1.Artifact) {
g.Expect(artifact.Metadata).To(BeEmpty())
},
},
{
name: "full reconcile - same rev, unobserved layer selector",
beforeFunc: func(obj *sourcev1.OCIRepository) {
obj.Spec.LayerSelector = &sourcev1.OCILayerSelector{
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Operation: sourcev1.OCILayerCopy,
}
obj.Status.Artifact = &sourcev1.Artifact{
Revision: testRevision,
}
@ -1320,10 +1345,13 @@ func TestOCIRepository_reconcileSource_noop(t *testing.T) {
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Operation: sourcev1.OCILayerCopy,
}
obj.Status.ObservedLayerSelector = &sourcev1.OCILayerSelector{
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Operation: sourcev1.OCILayerCopy,
}
obj.Status.Artifact = &sourcev1.Artifact{
Revision: testRevision,
}
obj.Status.ContentConfigChecksum = "sha256:fcfd705e10431a341f2df5b05ecee1fb54facd9a5e88b0be82276bdf533b6c64"
},
afterFunc: func(g *WithT, artifact *sourcev1.Artifact) {
g.Expect(artifact.Metadata).To(BeEmpty())
@ -1336,10 +1364,13 @@ func TestOCIRepository_reconcileSource_noop(t *testing.T) {
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Operation: sourcev1.OCILayerExtract,
}
obj.Status.ObservedLayerSelector = &sourcev1.OCILayerSelector{
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Operation: sourcev1.OCILayerCopy,
}
obj.Status.Artifact = &sourcev1.Artifact{
Revision: testRevision,
}
obj.Status.ContentConfigChecksum = "sha256:fcfd705e10431a341f2df5b05ecee1fb54facd9a5e88b0be82276bdf533b6c64"
},
afterFunc: func(g *WithT, artifact *sourcev1.Artifact) {
g.Expect(artifact.Metadata).ToNot(BeEmpty())
@ -1449,7 +1480,6 @@ func TestOCIRepository_reconcileArtifact(t *testing.T) {
obj.Status.Artifact = &sourcev1.Artifact{
Revision: "revision",
}
obj.Status.ContentConfigChecksum = ociRepoEmptyContentConfigChecksum
},
assertArtifact: &sourcev1.Artifact{
Revision: "revision",
@ -1467,14 +1497,13 @@ func TestOCIRepository_reconcileArtifact(t *testing.T) {
beforeFunc: func(obj *sourcev1.OCIRepository) {
obj.Status.Artifact = &sourcev1.Artifact{Revision: "revision"}
obj.Spec.Ignore = pointer.String("aaa")
obj.Status.ContentConfigChecksum = ociRepoEmptyContentConfigChecksum
},
want: sreconcile.ResultSuccess,
assertPaths: []string{
"latest.tar.gz",
},
afterFunc: func(g *WithT, obj *sourcev1.OCIRepository) {
g.Expect(obj.Status.ContentConfigChecksum).To(Equal("sha256:9834876dcfb05cb167a5c24953eba58c4ac89b1adf57f28f2f9d09af107ee8f0"))
g.Expect(*obj.Status.ObservedIgnore).To(Equal("aaa"))
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "stored artifact for digest"),
@ -1489,14 +1518,13 @@ func TestOCIRepository_reconcileArtifact(t *testing.T) {
beforeFunc: func(obj *sourcev1.OCIRepository) {
obj.Spec.LayerSelector = &sourcev1.OCILayerSelector{MediaType: "foo"}
obj.Status.Artifact = &sourcev1.Artifact{Revision: "revision"}
obj.Status.ContentConfigChecksum = ociRepoEmptyContentConfigChecksum
},
want: sreconcile.ResultSuccess,
assertPaths: []string{
"latest.tar.gz",
},
afterFunc: func(g *WithT, obj *sourcev1.OCIRepository) {
g.Expect(obj.Status.ContentConfigChecksum).To(Equal("sha256:82410edf339ab2945d97e26b92b6499e57156db63b94c17654b6ab97fbf86dbb"))
g.Expect(obj.Status.ObservedLayerSelector.MediaType).To(Equal("foo"))
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "stored artifact for digest"),
@ -1515,14 +1543,14 @@ func TestOCIRepository_reconcileArtifact(t *testing.T) {
Operation: sourcev1.OCILayerCopy,
}
obj.Status.Artifact = &sourcev1.Artifact{Revision: "revision"}
obj.Status.ContentConfigChecksum = ociRepoEmptyContentConfigChecksum
},
want: sreconcile.ResultSuccess,
assertPaths: []string{
"latest.tar.gz",
},
afterFunc: func(g *WithT, obj *sourcev1.OCIRepository) {
g.Expect(obj.Status.ContentConfigChecksum).To(Equal("sha256:0e0e1c82f6403c8ee74fdf51349c8b5d98c508b5374c507c7ffb2e41dbc875df"))
g.Expect(obj.Status.ObservedLayerSelector.MediaType).To(Equal("foo"))
g.Expect(obj.Status.ObservedLayerSelector.Operation).To(Equal(sourcev1.OCILayerCopy))
},
assertConditions: []metav1.Condition{
*conditions.TrueCondition(sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "stored artifact for digest"),
@ -1538,7 +1566,8 @@ func TestOCIRepository_reconcileArtifact(t *testing.T) {
obj.Spec.Ignore = pointer.String("aaa")
obj.Spec.LayerSelector = &sourcev1.OCILayerSelector{MediaType: "foo"}
obj.Status.Artifact = &sourcev1.Artifact{Revision: "revision"}
obj.Status.ContentConfigChecksum = "sha256:0b56187b81cab6c3485583a46bec631f5ea08a1f69b769457f0e4aafb47884e3"
obj.Status.ObservedIgnore = pointer.String("aaa")
obj.Status.ObservedLayerSelector = &sourcev1.OCILayerSelector{MediaType: "foo"}
},
want: sreconcile.ResultSuccess,
assertArtifact: &sourcev1.Artifact{
@ -2245,26 +2274,131 @@ func createTLSServer() (*httptest.Server, []byte, []byte, []byte, tls.Certificat
return srv, rootCertPEM, clientCertPEM, clientKeyPEM, clientTLSCert, err
}
func TestOCIRepository_calculateContentConfigChecksum(t *testing.T) {
g := NewWithT(t)
obj := &sourcev1.OCIRepository{}
r := &OCIRepositoryReconciler{}
emptyChecksum := r.calculateContentConfigChecksum(obj)
g.Expect(emptyChecksum).To(Equal(ociRepoEmptyContentConfigChecksum))
// Ignore modified.
obj.Spec.Ignore = pointer.String("some-rule")
ignoreModChecksum := r.calculateContentConfigChecksum(obj)
g.Expect(emptyChecksum).ToNot(Equal(ignoreModChecksum))
// LayerSelector modified.
obj.Spec.LayerSelector = &sourcev1.OCILayerSelector{
MediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
func TestOCIContentConfigChanged(t *testing.T) {
tests := []struct {
name string
spec sourcev1.OCIRepositorySpec
status sourcev1.OCIRepositoryStatus
want bool
}{
{
name: "same ignore, no layer selector",
spec: sourcev1.OCIRepositorySpec{
Ignore: pointer.String("nnn"),
},
status: sourcev1.OCIRepositoryStatus{
ObservedIgnore: pointer.String("nnn"),
},
want: false,
},
{
name: "different ignore, no layer selector",
spec: sourcev1.OCIRepositorySpec{
Ignore: pointer.String("nnn"),
},
status: sourcev1.OCIRepositoryStatus{
ObservedIgnore: pointer.String("mmm"),
},
want: true,
},
{
name: "same ignore, same layer selector",
spec: sourcev1.OCIRepositorySpec{
Ignore: pointer.String("nnn"),
LayerSelector: &sourcev1.OCILayerSelector{
MediaType: "foo",
Operation: sourcev1.OCILayerExtract,
},
},
status: sourcev1.OCIRepositoryStatus{
ObservedIgnore: pointer.String("nnn"),
ObservedLayerSelector: &sourcev1.OCILayerSelector{
MediaType: "foo",
Operation: sourcev1.OCILayerExtract,
},
},
want: false,
},
{
name: "same ignore, different layer selector operation",
spec: sourcev1.OCIRepositorySpec{
Ignore: pointer.String("nnn"),
LayerSelector: &sourcev1.OCILayerSelector{
MediaType: "foo",
Operation: sourcev1.OCILayerCopy,
},
},
status: sourcev1.OCIRepositoryStatus{
ObservedIgnore: pointer.String("nnn"),
ObservedLayerSelector: &sourcev1.OCILayerSelector{
MediaType: "foo",
Operation: sourcev1.OCILayerExtract,
},
},
want: true,
},
{
name: "same ignore, different layer selector mediatype",
spec: sourcev1.OCIRepositorySpec{
Ignore: pointer.String("nnn"),
LayerSelector: &sourcev1.OCILayerSelector{
MediaType: "bar",
Operation: sourcev1.OCILayerExtract,
},
},
status: sourcev1.OCIRepositoryStatus{
ObservedIgnore: pointer.String("nnn"),
ObservedLayerSelector: &sourcev1.OCILayerSelector{
MediaType: "foo",
Operation: sourcev1.OCILayerExtract,
},
},
want: true,
},
{
name: "no ignore, same layer selector",
spec: sourcev1.OCIRepositorySpec{
LayerSelector: &sourcev1.OCILayerSelector{
MediaType: "foo",
Operation: sourcev1.OCILayerExtract,
},
},
status: sourcev1.OCIRepositoryStatus{
ObservedLayerSelector: &sourcev1.OCILayerSelector{
MediaType: "foo",
Operation: sourcev1.OCILayerExtract,
},
},
want: false,
},
{
name: "no ignore, different layer selector",
spec: sourcev1.OCIRepositorySpec{
LayerSelector: &sourcev1.OCILayerSelector{
MediaType: "bar",
Operation: sourcev1.OCILayerExtract,
},
},
status: sourcev1.OCIRepositoryStatus{
ObservedLayerSelector: &sourcev1.OCILayerSelector{
MediaType: "foo",
Operation: sourcev1.OCILayerExtract,
},
},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
obj := &sourcev1.OCIRepository{
Spec: tt.spec,
Status: tt.status,
}
g.Expect(ociContentConfigChanged(obj)).To(Equal(tt.want))
})
}
mediaTypeChecksum := r.calculateContentConfigChecksum(obj)
g.Expect(ignoreModChecksum).ToNot(Equal(mediaTypeChecksum))
obj.Spec.LayerSelector.Operation = sourcev1.OCILayerCopy
layerCopyChecksum := r.calculateContentConfigChecksum(obj)
g.Expect(mediaTypeChecksum).ToNot(Equal(layerCopyChecksum))
}

92
docs/api/source.md
View File

@ -1518,6 +1518,19 @@ Artifact
</tr>
<tr>
<td>
<code>observedIgnore</code><br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>ObservedIgnore is the observed exclusion patterns used for constructing
the source artifact.</p>
</td>
</tr>
<tr>
<td>
<code>ReconcileRequestStatus</code><br>
<em>
<a href="https://godoc.org/github.com/fluxcd/pkg/apis/meta#ReconcileRequestStatus">
@ -1539,7 +1552,8 @@ github.com/fluxcd/pkg/apis/meta.ReconcileRequestStatus
</h3>
<p>
(<em>Appears on:</em>
<a href="#source.toolkit.fluxcd.io/v1beta2.GitRepositorySpec">GitRepositorySpec</a>)
<a href="#source.toolkit.fluxcd.io/v1beta2.GitRepositorySpec">GitRepositorySpec</a>,
<a href="#source.toolkit.fluxcd.io/v1beta2.GitRepositoryStatus">GitRepositoryStatus</a>)
</p>
<p>GitRepositoryInclude specifies a local reference to a GitRepository which
Artifact (sub-)contents must be included, and where they should be placed.</p>
@ -1969,6 +1983,49 @@ observed in .status.observedGeneration version of the object. This can
be used to determine if the content of the included repository has
changed.
It has the format of <code>&lt;algo&gt;:&lt;checksum&gt;</code>, for example: <code>sha256:&lt;checksum&gt;</code>.</p>
<p>Deprecated: Replaced with explicit fields for observed artifact content
config in the status.</p>
</td>
</tr>
<tr>
<td>
<code>observedIgnore</code><br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>ObservedIgnore is the observed exclusion patterns used for constructing
the source artifact.</p>
</td>
</tr>
<tr>
<td>
<code>observedRecurseSubmodules</code><br>
<em>
bool
</em>
</td>
<td>
<em>(Optional)</em>
<p>ObservedRecurseSubmodules is the observed resource submodules
configuration used to produce the current Artifact.</p>
</td>
</tr>
<tr>
<td>
<code>observedInclude</code><br>
<em>
<a href="#source.toolkit.fluxcd.io/v1beta2.GitRepositoryInclude">
[]GitRepositoryInclude
</a>
</em>
</td>
<td>
<em>(Optional)</em>
<p>ObservedInclude is the observed list of GitRepository resources used to
to produce the current Artifact.</p>
</td>
</tr>
<tr>
@ -2608,7 +2665,8 @@ string
</h3>
<p>
(<em>Appears on:</em>
<a href="#source.toolkit.fluxcd.io/v1beta2.OCIRepositorySpec">OCIRepositorySpec</a>)
<a href="#source.toolkit.fluxcd.io/v1beta2.OCIRepositorySpec">OCIRepositorySpec</a>,
<a href="#source.toolkit.fluxcd.io/v1beta2.OCIRepositoryStatus">OCIRepositoryStatus</a>)
</p>
<p>OCILayerSelector specifies which layer should be extracted from an OCI Artifact</p>
<div class="md-typeset__scrollwrap">
@ -3010,6 +3068,36 @@ observed in .status.observedGeneration version of the object. This can
be used to determine if the content configuration has changed and the
artifact needs to be rebuilt.
It has the format of <code>&lt;algo&gt;:&lt;checksum&gt;</code>, for example: <code>sha256:&lt;checksum&gt;</code>.</p>
<p>Deprecated: Replaced with explicit fields for observed artifact content
config in the status.</p>
</td>
</tr>
<tr>
<td>
<code>observedIgnore</code><br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>ObservedIgnore is the observed exclusion patterns used for constructing
the source artifact.</p>
</td>
</tr>
<tr>
<td>
<code>observedLayerSelector</code><br>
<em>
<a href="#source.toolkit.fluxcd.io/v1beta2.OCILayerSelector">
OCILayerSelector
</a>
</em>
</td>
<td>
<em>(Optional)</em>
<p>ObservedLayerSelector is the observed layer selector used for constructing
the source artifact.</p>
</td>
</tr>
<tr>

19
docs/spec/v1beta2/buckets.md
View File

@ -1064,6 +1064,25 @@ Note that a Bucket can be [reconciling](#reconciling-bucket) while failing at
the same time, for example due to a newly introduced configuration issue in the
Bucket spec.
### Observed Ignore
The source-controller reports an observed ignore in the Bucket's
`.status.observedIgnore`. The observed ignore is the latest `.spec.ignore` value
which resulted in a [ready state](#ready-bucket), or stalled due to error
it can not recover from without human intervention. The value is the same as the
[ignore in spec](#ignore). It indicates the ignore rules used in building the
current artifact in storage.
Example:
```yaml
status:
...
observedIgnore: |
hpa.yaml
build
...
```
### Observed Generation
The source-controller reports an

73
docs/spec/v1beta2/gitrepositories.md
View File

@ -854,6 +854,79 @@ configurations of the GitRepository that indicate a change in source and
records it in `.status.contentConfigChecksum`. This field is used to determine
if the source artifact needs to be rebuilt.
**Deprecation Note:** `contentConfigChecksum` is no longer used and will be
removed in the next API version. The individual components used for generating
content configuration checksum now have explicit fields in the status. This
makes the observations used by the controller for making artifact rebuild
decisions more transparent and easier to debug.
### Observed Ignore
The source-controller reports an observed ignore in the GitRepository's
`.status.observedIgnore`. The observed ignore is the latest `.spec.ignore` value
which resulted in a [ready state](#ready-gitrepository), or stalled due to error
it can not recover from without human intervention.
The value is the same as the [ignore in spec](#ignore).
It indicates the ignore rules used in building the current artifact in storage.
It is also used by the controller to determine if an artifact needs to be
rebuilt.
Example:
```yaml
status:
...
observedIgnore: |
cue
pkg
...
```
### Observed Recurse Submodules
The source-controller reports an observed recurse submodule in the
GitRepository's `.status.observedRecurseSubmodules`. The observed recurse
submodules is the latest `.spec.recurseSubmodules` value which resulted in a
[ready state](#ready-gitrepository), or stalled due to error it can not recover
from without human intervention. The value is the same as the
[recurse submodules in spec](#recurse-submodules). It indicates the recurse
submodules configuration used in building the current artifact in storage. It is
also used by the controller to determine if an artifact needs to be rebuilt.
Example:
```yaml
status:
...
observedRecurseSubmodules: true
...
```
### Observed Include
The source-controller reports observed include in the GitRepository's
`.status.observedInclude`. The observed include is the latest
`.spec.recurseSubmodules` value which resulted in a
[ready state](#ready-gitrepository), or stalled due to error it can not recover
from without human intervention. The value is the same as the
[include in spec](#include). It indicates the include configuration used in
building the current artifact in storage. It is also used by the controller to
determine if an artifact needs to be rebuilt.
Example:
```yaml
status:
...
observedInclude:
- fromPath: deploy/webapp
repository:
name: repo1
toPath: foo
- fromPath: deploy/secure
repository:
name: repo2
toPath: bar
...
```
### Observed Generation
The source-controller reports an [observed generation][typical-status-properties]

47
docs/spec/v1beta2/ocirepositories.md
View File

@ -868,6 +868,53 @@ configurations of the OCIRepository that indicate a change in source and
records it in `.status.contentConfigChecksum`. This field is used to determine
if the source artifact needs to be rebuilt.
**Deprecation Note:** `contentConfigChecksum` is no longer used and will be
removed in the next API version. The individual components used for generating
content configuration checksum now have explicit fields in the status. This
makes the observations used by the controller for making artifact rebuild
decisions more transparent and easier to debug.
### Observed Ignore
The source-controller reports an observed ignore in the OCIRepository's
`.status.observedIgnore`. The observed ignore is the latest `.spec.ignore` value
which resulted in a [ready state](#ready-ocirepository), or stalled due to error
it can not recover from without human intervention. The value is the same as the
[ignore in spec](#ignore). It indicates the ignore rules used in building the
current artifact in storage. It is also used by the controller to determine if
an artifact needs to be rebuilt.
Example:
```yaml
status:
...
observedIgnore: |
hpa.yaml
build
...
```
### Observed Layer Selector
The source-controller reports an observed layer selector in the OCIRepository's
`.status.observedLayerSelector`. The observed layer selector is the latest
`.spec.layerSelector` value which resulted in a [ready state](#ready-ocirepository),
or stalled due to error it can not recover from without human intervention.
The value is the same as the [layer selector in spec](#layer-selector).
It indicates the layer selection configuration used in building the current
artifact in storage. It is also used by the controller to determine if an
artifact needs to be rebuilt.
Example:
```yaml
status:
...
observedLayerSelector:
mediaType: application/vnd.docker.image.rootfs.diff.tar.gzip
operation: copy
...
```
### Observed Generation
The source-controller reports an [observed generation][typical-status-properties]