Emit events for garbage collection
This commit is contained in:
parent
406ce977a4
commit
4d3a3a7bae
|
|
@ -451,7 +451,7 @@ func (r *KustomizationReconciler) validate(kustomization kustomizev1.Kustomizati
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization, revision, dirPath string) error {
|
func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization, revision, dirPath string) (string, error) {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
timeout := kustomization.GetTimeout() + (time.Second * 1)
|
timeout := kustomization.GetTimeout() + (time.Second * 1)
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||||
|
|
@ -469,9 +469,9 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
|
||||||
output, err := command.CombinedOutput()
|
output, err := command.CombinedOutput()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, context.DeadlineExceeded) {
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
return fmt.Errorf("apply timeout: %w", err)
|
return "", fmt.Errorf("apply timeout: %w", err)
|
||||||
}
|
}
|
||||||
return fmt.Errorf("apply failed: %s", string(output))
|
return "", fmt.Errorf("apply failed: %s", string(output))
|
||||||
}
|
}
|
||||||
|
|
||||||
resources := r.parseApplyOutput(output)
|
resources := r.parseApplyOutput(output)
|
||||||
|
|
@ -484,22 +484,17 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
|
||||||
"output", resources,
|
"output", resources,
|
||||||
)
|
)
|
||||||
|
|
||||||
var diff bool
|
|
||||||
changeSet := ""
|
changeSet := ""
|
||||||
for obj, action := range resources {
|
for obj, action := range resources {
|
||||||
if action != "" && action != "unchanged" {
|
if action != "" && action != "unchanged" {
|
||||||
diff = true
|
|
||||||
changeSet += obj + " " + action + "\n"
|
changeSet += obj + " " + action + "\n"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if diff {
|
return changeSet, nil
|
||||||
r.event(kustomization, revision, recorder.EventSeverityInfo, changeSet)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kustomization, revision, dirPath string, delay time.Duration) error {
|
func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kustomization, revision, dirPath string, delay time.Duration) error {
|
||||||
err := r.apply(kustomization, revision, dirPath)
|
changeSet, err := r.apply(kustomization, revision, dirPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// retry apply due to CRD/CR race
|
// retry apply due to CRD/CR race
|
||||||
if strings.Contains(err.Error(), "could not find the requested resource") ||
|
if strings.Contains(err.Error(), "could not find the requested resource") ||
|
||||||
|
|
@ -508,12 +503,20 @@ func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kusto
|
||||||
"error", err.Error(),
|
"error", err.Error(),
|
||||||
"kustomization", fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()))
|
"kustomization", fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()))
|
||||||
time.Sleep(delay)
|
time.Sleep(delay)
|
||||||
if err := r.apply(kustomization, revision, dirPath); err != nil {
|
if changeSet, err := r.apply(kustomization, revision, dirPath); err != nil {
|
||||||
return err
|
return err
|
||||||
|
} else {
|
||||||
|
if changeSet != "" {
|
||||||
|
r.event(kustomization, revision, recorder.EventSeverityInfo, changeSet)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
if changeSet != "" {
|
||||||
|
r.event(kustomization, revision, recorder.EventSeverityInfo, changeSet)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
@ -526,13 +529,25 @@ func (r *KustomizationReconciler) prune(kustomization kustomizev1.Kustomization,
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if !prune(kustomization.GetTimeout(),
|
if output, ok := prune(kustomization.GetTimeout(),
|
||||||
kustomization.GetName(),
|
kustomization.GetName(),
|
||||||
kustomization.GetNamespace(),
|
kustomization.GetNamespace(),
|
||||||
kustomization.Status.Snapshot,
|
kustomization.Status.Snapshot,
|
||||||
r.Log,
|
r.Log,
|
||||||
) {
|
); !ok {
|
||||||
return fmt.Errorf("pruning failed")
|
return fmt.Errorf("pruning failed")
|
||||||
|
} else {
|
||||||
|
changeSet := ""
|
||||||
|
input := strings.Split(output, "\n")
|
||||||
|
for _, action := range input {
|
||||||
|
if strings.Contains(action, "deleted") {
|
||||||
|
changeSet += action + "\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if changeSet != "" {
|
||||||
|
r.event(kustomization, snapshot.Revision, recorder.EventSeverityInfo, changeSet)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
@ -645,7 +660,11 @@ func (r *KustomizationReconciler) event(kustomization kustomizev1.Kustomization,
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.ExternalEventRecorder != nil {
|
if r.ExternalEventRecorder != nil {
|
||||||
if err := r.ExternalEventRecorder.Eventf(*objRef, map[string]string{"revision": revision}, severity, severity, msg); err != nil {
|
var meta map[string]string
|
||||||
|
if revision != "" {
|
||||||
|
meta = map[string]string{"revision": revision}
|
||||||
|
}
|
||||||
|
if err := r.ExternalEventRecorder.Eventf(*objRef, meta, severity, severity, msg); err != nil {
|
||||||
r.Log.WithValues(
|
r.Log.WithValues(
|
||||||
strings.ToLower(kustomization.Kind),
|
strings.ToLower(kustomization.Kind),
|
||||||
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
|
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
|
||||||
|
|
|
||||||
|
|
@ -78,9 +78,10 @@ func (gc KustomizationGarbageCollectPredicate) Delete(e event.DeleteEvent) bool
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func prune(timeout time.Duration, name string, namespace string, snapshot *kustomizev1.Snapshot, log logr.Logger) bool {
|
func prune(timeout time.Duration, name string, namespace string, snapshot *kustomizev1.Snapshot, log logr.Logger) (string, bool) {
|
||||||
selector := gcSelectors(name, namespace, snapshot.Revision)
|
selector := gcSelectors(name, namespace, snapshot.Revision)
|
||||||
ok := true
|
ok := true
|
||||||
|
changeSet := ""
|
||||||
outInfo := ""
|
outInfo := ""
|
||||||
outErr := ""
|
outErr := ""
|
||||||
for ns, kinds := range snapshot.NamespacedKinds() {
|
for ns, kinds := range snapshot.NamespacedKinds() {
|
||||||
|
|
@ -89,7 +90,7 @@ func prune(timeout time.Duration, name string, namespace string, snapshot *kusto
|
||||||
outErr += " " + err.Error()
|
outErr += " " + err.Error()
|
||||||
ok = false
|
ok = false
|
||||||
} else {
|
} else {
|
||||||
outInfo += " " + output
|
outInfo += " " + output + "\n"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -97,6 +98,7 @@ func prune(timeout time.Duration, name string, namespace string, snapshot *kusto
|
||||||
log.Info("Garbage collection for namespaced objects completed",
|
log.Info("Garbage collection for namespaced objects completed",
|
||||||
"kustomization", fmt.Sprintf("%s/%s", namespace, name),
|
"kustomization", fmt.Sprintf("%s/%s", namespace, name),
|
||||||
"output", outInfo)
|
"output", outInfo)
|
||||||
|
changeSet += outInfo
|
||||||
} else {
|
} else {
|
||||||
log.Error(fmt.Errorf(outErr), "Garbage collection for namespaced objects failed",
|
log.Error(fmt.Errorf(outErr), "Garbage collection for namespaced objects failed",
|
||||||
"kustomization", fmt.Sprintf("%s/%s", namespace, name))
|
"kustomization", fmt.Sprintf("%s/%s", namespace, name))
|
||||||
|
|
@ -109,19 +111,20 @@ func prune(timeout time.Duration, name string, namespace string, snapshot *kusto
|
||||||
outErr += " " + err.Error()
|
outErr += " " + err.Error()
|
||||||
ok = false
|
ok = false
|
||||||
} else {
|
} else {
|
||||||
outInfo += " " + output
|
outInfo += " " + output + "\n"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if outErr == "" {
|
if outErr == "" {
|
||||||
log.Info("Garbage collection for non-namespaced objects completed",
|
log.Info("Garbage collection for non-namespaced objects completed",
|
||||||
"kustomization", fmt.Sprintf("%s/%s", namespace, name),
|
"kustomization", fmt.Sprintf("%s/%s", namespace, name),
|
||||||
"output", outInfo)
|
"output", outInfo)
|
||||||
|
changeSet += outInfo
|
||||||
} else {
|
} else {
|
||||||
log.Error(fmt.Errorf(outErr), "Garbage collection for non-namespaced objects failed",
|
log.Error(fmt.Errorf(outErr), "Garbage collection for non-namespaced objects failed",
|
||||||
"kustomization", fmt.Sprintf("%s/%s", namespace, name))
|
"kustomization", fmt.Sprintf("%s/%s", namespace, name))
|
||||||
}
|
}
|
||||||
|
|
||||||
return ok
|
return changeSet, ok
|
||||||
}
|
}
|
||||||
|
|
||||||
func deleteByKind(timeout time.Duration, kind, namespace, selector string) (string, error) {
|
func deleteByKind(timeout time.Duration, kind, namespace, selector string) (string, error) {
|
||||||
|
|
|
||||||
Binary file not shown.
|
Before Width: | Height: | Size: 14 KiB After Width: | Height: | Size: 34 KiB |
|
|
@ -190,22 +190,4 @@ spec:
|
||||||
validate: server
|
validate: server
|
||||||
```
|
```
|
||||||
|
|
||||||
Configure alerting for all pipelines in the `kustomize-system` namespace:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
apiVersion: kustomize.fluxcd.io/v1alpha1
|
|
||||||
kind: Profile
|
|
||||||
metadata:
|
|
||||||
name: default
|
|
||||||
namespace: kustomize-system
|
|
||||||
spec:
|
|
||||||
alert:
|
|
||||||
type: slack
|
|
||||||
verbosity: info
|
|
||||||
address: https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK
|
|
||||||
username: kustomize-controller
|
|
||||||
channel: general
|
|
||||||
kustomizations:
|
|
||||||
- '*'
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue