fluxcd
/
kustomize-controller
mirror of https://github.com/fluxcd/kustomize-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1502 from fluxcd/status-history 

Track reconciliation attempts over time in `.status.history`
This commit is contained in:
Stefan Prodan 2025-09-01 10:46:49 +03:00 committed by GitHub
parent 6b2e5e8729 cf3e7b7204
commit ca604f3389
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 165 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/v1/kustomization_types.go
View File

@ -297,6 +297,11 @@ type KustomizationStatus struct {
// have been successfully applied.
// +optional
Inventory *ResourceInventory `json:"inventory,omitempty"`
// History contains a set of snapshots of the last reconciliation attempts
// tracking the revision, the state and the duration of each attempt.
// +optional
History meta.History `json:"history,omitempty"`
}
// GetTimeout returns the timeout with default.

7
api/v1/zz_generated.deepcopy.go
View File

@ -260,6 +260,13 @@ func (in *KustomizationStatus) DeepCopyInto(out *KustomizationStatus) {
*out = new(ResourceInventory)
(*in).DeepCopyInto(*out)
}
if in.History != nil {
in, out := &in.History, &out.History
*out = make(meta.History, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KustomizationStatus.

51
config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml
View File

@ -591,6 +591,57 @@ spec:
- type
type: object
type: array
history:
description: |-
History contains a set of snapshots of the last reconciliation attempts
tracking the revision, the state and the duration of each attempt.
items:
description: |-
Snapshot represents a point-in-time record of a group of resources reconciliation,
including timing information, status, and a unique digest identifier.
properties:
digest:
description: Digest is the checksum in the format `<algo>:<hex>`
of the resources in this snapshot.
type: string
firstReconciled:
description: FirstReconciled is the time when this revision
was first reconciled to the cluster.
format: date-time
type: string
lastReconciled:
description: LastReconciled is the time when this revision was
last reconciled to the cluster.
format: date-time
type: string
lastReconciledDuration:
description: LastReconciledDuration is time it took to reconcile
the resources in this revision.
type: string
lastReconciledStatus:
description: LastReconciledStatus is the status of the last
reconciliation.
type: string
metadata:
additionalProperties:
type: string
description: Metadata contains additional information about
the snapshot.
type: object
totalReconciliations:
description: TotalReconciliations is the total number of reconciliations
that have occurred for this snapshot.
format: int64
type: integer
required:
- digest
- firstReconciled
- lastReconciled
- lastReconciledDuration
- lastReconciledStatus
- totalReconciliations
type: object
type: array
inventory:
description: |-
Inventory contains the list of Kubernetes resource object references that

15
docs/api/v1/kustomize.md
View File

@ -1147,6 +1147,21 @@ ResourceInventory
have been successfully applied.</p>
</td>
</tr>
<tr>
<td>
<code>history</code><br>
<em>
<a href="https://godoc.org/github.com/fluxcd/pkg/apis/meta#History">
github.com/fluxcd/pkg/apis/meta.History
</a>
</em>
</td>
<td>
<em>(Optional)</em>
<p>History contains a set of snapshots of the last reconciliation attempts
tracking the revision, the state and the duration of each attempt.</p>
</td>
</tr>
</tbody>
</table>
</div>

31
docs/spec/v1/kustomizations.md
View File

@ -2066,6 +2066,37 @@ configuration issue in the Kustomization spec. When a reconciliation fails, the
`Reconciling` Condition `reason` would be `ProgressingWithRetry`. When the
reconciliation is performed again after the failure, the `reason` is updated to `Progressing`.
### History
The kustomize-controller maintains a history of the last 5 reconciliations
in `.status.history`, including the digest of the applied manifests, the
source and origin revision, the timestamps and the duration of the reconciliations,
the status and the total number of times a specific digest was reconciled.
```yaml
status:
history:
- digest: sha256:43ad78c94b2655429d84f21488f29d7cca9cd45b7f54d2b27e16bbec8eff9228
firstReconciled: "2025-08-15T10:11:00Z"
lastReconciled: "2025-08-15T11:12:00Z"
lastReconciledDuration: 2.818583s
lastReconciledStatus: ReconciliationSucceeded
totalReconciliations: 2
metadata:
revision: "v1.0.1@sha1:450796ddb2ab6724ee1cc32a4be56da032d1cca0"
- digest: sha256:ec8dbfe61777b65001190260cf873ffe454451bd2e464bd6f9a154cffcdcd7e5
firstReconciled: "2025-07-14T13:10:00Z"
lastReconciled: "2025-08-15T10:00:00Z"
lastReconciledDuration: 49.813292s
lastReconciledStatus: HealthCheckFailed
totalReconciliations: 120
metadata:
revision: "v1.0.0@sha1:67e2c98a60dc92283531412a9e604dd4bae005a9"
```
The kustomize-controller deduplicates entries based on the digest and status, with the
most recent reconciliation being the first entry in the list.
### Inventory
In order to perform operations such as drift detection, garbage collection, etc.

34
internal/controller/kustomization_controller.go
View File

@ -27,6 +27,7 @@ import (
securejoin "github.com/cyphar/filepath-securejoin"
celtypes "github.com/google/cel-go/common/types"
"github.com/opencontainers/go-digest"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
@ -281,6 +282,7 @@ func (r *KustomizationReconciler) reconcile(
src sourcev1.Source,
patcher *patch.SerialPatcher,
statusReaders []func(apimeta.RESTMapper) engine.StatusReader) error {
reconcileStart := time.Now()
log := ctrl.LoggerFrom(ctx)
// Update status with the reconciliation progress.
@ -314,13 +316,14 @@ func (r *KustomizationReconciler) reconcile(
}(tmpDir)
// Download artifact and extract files to the tmp dir.
if err = fetch.NewArchiveFetcherWithLogger(
r.artifactFetchRetries,
tar.UnlimitedUntarSize,
tar.UnlimitedUntarSize,
os.Getenv("SOURCE_CONTROLLER_LOCALHOST"),
ctrl.LoggerFrom(ctx),
).Fetch(src.GetArtifact().URL, src.GetArtifact().Digest, tmpDir); err != nil {
fetcher := fetch.New(
fetch.WithLogger(ctrl.LoggerFrom(ctx)),
fetch.WithRetries(r.artifactFetchRetries),
fetch.WithMaxDownloadSize(tar.UnlimitedUntarSize),
fetch.WithUntar(tar.WithMaxUntarSize(tar.UnlimitedUntarSize)),
fetch.WithHostnameOverwrite(os.Getenv("SOURCE_CONTROLLER_LOCALHOST")),
)
if err = fetcher.Fetch(src.GetArtifact().URL, src.GetArtifact().Digest, tmpDir); err != nil {
conditions.MarkFalse(obj, meta.ReadyCondition, meta.ArtifactFailedReason, "%s", err)
return err
}
@ -398,6 +401,13 @@ func (r *KustomizationReconciler) reconcile(
return err
}
// Calculate the digest of the built resources for history tracking.
checksum := digest.FromBytes(resources).String()
historyMeta := map[string]string{"revision": revision}
if originRevision != "" {
historyMeta["originRevision"] = originRevision
}
// Convert the build result into Kubernetes unstructured objects.
objects, err := ssautil.ReadObjects(bytes.NewReader(resources))
if err != nil {
@ -423,6 +433,7 @@ func (r *KustomizationReconciler) reconcile(
// Validate and apply resources in stages.
drifted, changeSet, err := r.apply(ctx, resourceManager, obj, revision, originRevision, objects)
if err != nil {
obj.Status.History.Upsert(checksum, time.Now(), time.Since(reconcileStart), meta.ReconciliationFailedReason, historyMeta)
conditions.MarkFalse(obj, meta.ReadyCondition, meta.ReconciliationFailedReason, "%s", err)
return err
}
@ -431,6 +442,7 @@ func (r *KustomizationReconciler) reconcile(
newInventory := inventory.New()
err = inventory.AddChangeSet(newInventory, changeSet)
if err != nil {
obj.Status.History.Upsert(checksum, time.Now(), time.Since(reconcileStart), meta.ReconciliationFailedReason, historyMeta)
conditions.MarkFalse(obj, meta.ReadyCondition, meta.ReconciliationFailedReason, "%s", err)
return err
}
@ -441,12 +453,14 @@ func (r *KustomizationReconciler) reconcile(
// Detect stale resources which are subject to garbage collection.
staleObjects, err := inventory.Diff(oldInventory, newInventory)
if err != nil {
obj.Status.History.Upsert(checksum, time.Now(), time.Since(reconcileStart), meta.ReconciliationFailedReason, historyMeta)
conditions.MarkFalse(obj, meta.ReadyCondition, meta.ReconciliationFailedReason, "%s", err)
return err
}
// Run garbage collection for stale resources that do not have pruning disabled.
if _, err := r.prune(ctx, resourceManager, obj, revision, originRevision, staleObjects); err != nil {
obj.Status.History.Upsert(checksum, time.Now(), time.Since(reconcileStart), meta.PruneFailedReason, historyMeta)
conditions.MarkFalse(obj, meta.ReadyCondition, meta.PruneFailedReason, "%s", err)
return err
}
@ -462,6 +476,7 @@ func (r *KustomizationReconciler) reconcile(
isNewRevision,
drifted,
changeSet.ToObjMetadataSet()); err != nil {
obj.Status.History.Upsert(checksum, time.Now(), time.Since(reconcileStart), meta.HealthCheckFailedReason, historyMeta)
conditions.MarkFalse(obj, meta.ReadyCondition, meta.HealthCheckFailedReason, "%s", err)
return err
}
@ -475,6 +490,11 @@ func (r *KustomizationReconciler) reconcile(
meta.ReadyCondition,
meta.ReconciliationSucceededReason,
"Applied revision: %s", revision)
obj.Status.History.Upsert(checksum,
time.Now(),
time.Since(reconcileStart),
meta.ReconciliationSucceededReason,
historyMeta)
return nil
}

9
internal/controller/kustomization_origin_revision_test.go
View File

@ -115,6 +115,15 @@ stringData:
g.Expect(resultK.Status.LastAppliedOriginRevision).To(Equal("orev"))
g.Expect(resultK.Status.History).To(HaveLen(1))
g.Expect(resultK.Status.History[0].Digest).ToNot(BeEmpty())
g.Expect(resultK.Status.History[0].TotalReconciliations).To(BeEquivalentTo(1))
g.Expect(resultK.Status.History[0].FirstReconciled.Time).ToNot(BeZero())
g.Expect(resultK.Status.History[0].LastReconciled.Time).ToNot(BeZero())
g.Expect(resultK.Status.History[0].LastReconciledStatus).To(Equal(meta.ReconciliationSucceededReason))
g.Expect(resultK.Status.History[0].LastReconciledDuration.Duration).To(BeNumerically(">", 0))
g.Expect(resultK.Status.History[0].Metadata).To(ContainElements(revision, "orev"))
events := getEvents(kustomizationKey.Name, nil)
g.Expect(events).To(Not(BeEmpty()))

20
internal/controller/kustomization_wait_test.go
View File

@ -136,6 +136,11 @@ parameters:
g.Expect(resultK.Status.ObservedGeneration).To(BeIdenticalTo(resultK.Generation))
kstatusCheck.CheckErr(ctx, resultK)
g.Expect(resultK.Status.History).To(HaveLen(1))
g.Expect(resultK.Status.History[0].TotalReconciliations).To(BeEquivalentTo(1))
g.Expect(resultK.Status.History[0].LastReconciledStatus).To(Equal(meta.ReconciliationSucceededReason))
g.Expect(resultK.Status.History[0].Metadata).To(ContainElements(revision))
})
t.Run("reports progressing status", func(t *testing.T) {
@ -192,6 +197,11 @@ parameters:
g.Expect(resultK.Status.ObservedGeneration).To(BeIdenticalTo(resultK.Generation - 1))
kstatusCheck.CheckErr(ctx, resultK)
g.Expect(resultK.Status.History).To(HaveLen(2))
g.Expect(resultK.Status.History[0].TotalReconciliations).To(BeEquivalentTo(1))
g.Expect(resultK.Status.History[0].LastReconciledStatus).To(Equal(meta.HealthCheckFailedReason))
g.Expect(resultK.Status.History[0].Metadata).To(ContainElements(revision))
})
t.Run("emits unhealthy event", func(t *testing.T) {
@ -228,6 +238,11 @@ parameters:
g.Expect(resultK.Status.ObservedGeneration).To(BeIdenticalTo(resultK.Generation))
kstatusCheck.CheckErr(ctx, resultK)
g.Expect(resultK.Status.History).To(HaveLen(2))
g.Expect(resultK.Status.History[0].TotalReconciliations).To(BeEquivalentTo(2))
g.Expect(resultK.Status.History[0].LastReconciledStatus).To(Equal(meta.ReconciliationSucceededReason))
g.Expect(resultK.Status.History[0].Metadata).To(ContainElements(revision))
})
t.Run("emits recovery event", func(t *testing.T) {
@ -258,6 +273,11 @@ parameters:
g.Expect(resultK.Status.LastAttemptedRevision).To(BeIdenticalTo(resultK.Status.LastAppliedRevision))
kstatusCheck.CheckErr(ctx, resultK)
g.Expect(resultK.Status.History).To(HaveLen(3))
g.Expect(resultK.Status.History[0].TotalReconciliations).To(BeEquivalentTo(1))
g.Expect(resultK.Status.History[0].LastReconciledStatus).To(Equal(meta.ReconciliationSucceededReason))
g.Expect(resultK.Status.History[0].Metadata).To(ContainElements(revision))
})
t.Run("emits event for the new revision", func(t *testing.T) {