Emit events for garbage collection

This commit is contained in:
stefanprodan 2020-07-02 08:40:55 +03:00
parent 406ce977a4
commit 4d3a3a7bae
4 changed files with 40 additions and 36 deletions

View File

@ -451,7 +451,7 @@ func (r *KustomizationReconciler) validate(kustomization kustomizev1.Kustomizati
return nil return nil
} }
func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization, revision, dirPath string) error { func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization, revision, dirPath string) (string, error) {
start := time.Now() start := time.Now()
timeout := kustomization.GetTimeout() + (time.Second * 1) timeout := kustomization.GetTimeout() + (time.Second * 1)
ctx, cancel := context.WithTimeout(context.Background(), timeout) ctx, cancel := context.WithTimeout(context.Background(), timeout)
@ -469,9 +469,9 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
output, err := command.CombinedOutput() output, err := command.CombinedOutput()
if err != nil { if err != nil {
if errors.Is(err, context.DeadlineExceeded) { if errors.Is(err, context.DeadlineExceeded) {
return fmt.Errorf("apply timeout: %w", err) return "", fmt.Errorf("apply timeout: %w", err)
} }
return fmt.Errorf("apply failed: %s", string(output)) return "", fmt.Errorf("apply failed: %s", string(output))
} }
resources := r.parseApplyOutput(output) resources := r.parseApplyOutput(output)
@ -484,22 +484,17 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
"output", resources, "output", resources,
) )
var diff bool
changeSet := "" changeSet := ""
for obj, action := range resources { for obj, action := range resources {
if action != "" && action != "unchanged" { if action != "" && action != "unchanged" {
diff = true
changeSet += obj + " " + action + "\n" changeSet += obj + " " + action + "\n"
} }
} }
if diff { return changeSet, nil
r.event(kustomization, revision, recorder.EventSeverityInfo, changeSet)
}
return nil
} }
func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kustomization, revision, dirPath string, delay time.Duration) error { func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kustomization, revision, dirPath string, delay time.Duration) error {
err := r.apply(kustomization, revision, dirPath) changeSet, err := r.apply(kustomization, revision, dirPath)
if err != nil { if err != nil {
// retry apply due to CRD/CR race // retry apply due to CRD/CR race
if strings.Contains(err.Error(), "could not find the requested resource") || if strings.Contains(err.Error(), "could not find the requested resource") ||
@ -508,12 +503,20 @@ func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kusto
"error", err.Error(), "error", err.Error(),
"kustomization", fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName())) "kustomization", fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()))
time.Sleep(delay) time.Sleep(delay)
if err := r.apply(kustomization, revision, dirPath); err != nil { if changeSet, err := r.apply(kustomization, revision, dirPath); err != nil {
return err return err
} else {
if changeSet != "" {
r.event(kustomization, revision, recorder.EventSeverityInfo, changeSet)
}
} }
} else { } else {
return err return err
} }
} else {
if changeSet != "" {
r.event(kustomization, revision, recorder.EventSeverityInfo, changeSet)
}
} }
return nil return nil
} }
@ -526,13 +529,25 @@ func (r *KustomizationReconciler) prune(kustomization kustomizev1.Kustomization,
return nil return nil
} }
if !prune(kustomization.GetTimeout(), if output, ok := prune(kustomization.GetTimeout(),
kustomization.GetName(), kustomization.GetName(),
kustomization.GetNamespace(), kustomization.GetNamespace(),
kustomization.Status.Snapshot, kustomization.Status.Snapshot,
r.Log, r.Log,
) { ); !ok {
return fmt.Errorf("pruning failed") return fmt.Errorf("pruning failed")
} else {
changeSet := ""
input := strings.Split(output, "\n")
for _, action := range input {
if strings.Contains(action, "deleted") {
changeSet += action + "\n"
}
}
if changeSet != "" {
r.event(kustomization, snapshot.Revision, recorder.EventSeverityInfo, changeSet)
}
} }
return nil return nil
} }
@ -645,7 +660,11 @@ func (r *KustomizationReconciler) event(kustomization kustomizev1.Kustomization,
} }
if r.ExternalEventRecorder != nil { if r.ExternalEventRecorder != nil {
if err := r.ExternalEventRecorder.Eventf(*objRef, map[string]string{"revision": revision}, severity, severity, msg); err != nil { var meta map[string]string
if revision != "" {
meta = map[string]string{"revision": revision}
}
if err := r.ExternalEventRecorder.Eventf(*objRef, meta, severity, severity, msg); err != nil {
r.Log.WithValues( r.Log.WithValues(
strings.ToLower(kustomization.Kind), strings.ToLower(kustomization.Kind),
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()), fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),

View File

@ -78,9 +78,10 @@ func (gc KustomizationGarbageCollectPredicate) Delete(e event.DeleteEvent) bool
return true return true
} }
func prune(timeout time.Duration, name string, namespace string, snapshot *kustomizev1.Snapshot, log logr.Logger) bool { func prune(timeout time.Duration, name string, namespace string, snapshot *kustomizev1.Snapshot, log logr.Logger) (string, bool) {
selector := gcSelectors(name, namespace, snapshot.Revision) selector := gcSelectors(name, namespace, snapshot.Revision)
ok := true ok := true
changeSet := ""
outInfo := "" outInfo := ""
outErr := "" outErr := ""
for ns, kinds := range snapshot.NamespacedKinds() { for ns, kinds := range snapshot.NamespacedKinds() {
@ -89,7 +90,7 @@ func prune(timeout time.Duration, name string, namespace string, snapshot *kusto
outErr += " " + err.Error() outErr += " " + err.Error()
ok = false ok = false
} else { } else {
outInfo += " " + output outInfo += " " + output + "\n"
} }
} }
} }
@ -97,6 +98,7 @@ func prune(timeout time.Duration, name string, namespace string, snapshot *kusto
log.Info("Garbage collection for namespaced objects completed", log.Info("Garbage collection for namespaced objects completed",
"kustomization", fmt.Sprintf("%s/%s", namespace, name), "kustomization", fmt.Sprintf("%s/%s", namespace, name),
"output", outInfo) "output", outInfo)
changeSet += outInfo
} else { } else {
log.Error(fmt.Errorf(outErr), "Garbage collection for namespaced objects failed", log.Error(fmt.Errorf(outErr), "Garbage collection for namespaced objects failed",
"kustomization", fmt.Sprintf("%s/%s", namespace, name)) "kustomization", fmt.Sprintf("%s/%s", namespace, name))
@ -109,19 +111,20 @@ func prune(timeout time.Duration, name string, namespace string, snapshot *kusto
outErr += " " + err.Error() outErr += " " + err.Error()
ok = false ok = false
} else { } else {
outInfo += " " + output outInfo += " " + output + "\n"
} }
} }
if outErr == "" { if outErr == "" {
log.Info("Garbage collection for non-namespaced objects completed", log.Info("Garbage collection for non-namespaced objects completed",
"kustomization", fmt.Sprintf("%s/%s", namespace, name), "kustomization", fmt.Sprintf("%s/%s", namespace, name),
"output", outInfo) "output", outInfo)
changeSet += outInfo
} else { } else {
log.Error(fmt.Errorf(outErr), "Garbage collection for non-namespaced objects failed", log.Error(fmt.Errorf(outErr), "Garbage collection for non-namespaced objects failed",
"kustomization", fmt.Sprintf("%s/%s", namespace, name)) "kustomization", fmt.Sprintf("%s/%s", namespace, name))
} }
return ok return changeSet, ok
} }
func deleteByKind(timeout time.Duration, kind, namespace, selector string) (string, error) { func deleteByKind(timeout time.Duration, kind, namespace, selector string) (string, error) {

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 34 KiB

View File

@ -190,22 +190,4 @@ spec:
validate: server validate: server
``` ```
Configure alerting for all pipelines in the `kustomize-system` namespace:
```yaml
apiVersion: kustomize.fluxcd.io/v1alpha1
kind: Profile
metadata:
name: default
namespace: kustomize-system
spec:
alert:
type: slack
verbosity: info
address: https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK
username: kustomize-controller
channel: general
kustomizations:
- '*'
```