Emit events for garbage collection

This commit is contained in:
stefanprodan 2020-07-02 08:40:55 +03:00
parent 406ce977a4
commit 4d3a3a7bae
4 changed files with 40 additions and 36 deletions

View File

@ -451,7 +451,7 @@ func (r *KustomizationReconciler) validate(kustomization kustomizev1.Kustomizati
return nil
}
func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization, revision, dirPath string) error {
func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization, revision, dirPath string) (string, error) {
start := time.Now()
timeout := kustomization.GetTimeout() + (time.Second * 1)
ctx, cancel := context.WithTimeout(context.Background(), timeout)
@ -469,9 +469,9 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
output, err := command.CombinedOutput()
if err != nil {
if errors.Is(err, context.DeadlineExceeded) {
return fmt.Errorf("apply timeout: %w", err)
return "", fmt.Errorf("apply timeout: %w", err)
}
return fmt.Errorf("apply failed: %s", string(output))
return "", fmt.Errorf("apply failed: %s", string(output))
}
resources := r.parseApplyOutput(output)
@ -484,22 +484,17 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
"output", resources,
)
var diff bool
changeSet := ""
for obj, action := range resources {
if action != "" && action != "unchanged" {
diff = true
changeSet += obj + " " + action + "\n"
}
}
if diff {
r.event(kustomization, revision, recorder.EventSeverityInfo, changeSet)
}
return nil
return changeSet, nil
}
func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kustomization, revision, dirPath string, delay time.Duration) error {
err := r.apply(kustomization, revision, dirPath)
changeSet, err := r.apply(kustomization, revision, dirPath)
if err != nil {
// retry apply due to CRD/CR race
if strings.Contains(err.Error(), "could not find the requested resource") ||
@ -508,12 +503,20 @@ func (r *KustomizationReconciler) applyWithRetry(kustomization kustomizev1.Kusto
"error", err.Error(),
"kustomization", fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()))
time.Sleep(delay)
if err := r.apply(kustomization, revision, dirPath); err != nil {
if changeSet, err := r.apply(kustomization, revision, dirPath); err != nil {
return err
} else {
if changeSet != "" {
r.event(kustomization, revision, recorder.EventSeverityInfo, changeSet)
}
}
} else {
return err
}
} else {
if changeSet != "" {
r.event(kustomization, revision, recorder.EventSeverityInfo, changeSet)
}
}
return nil
}
@ -526,13 +529,25 @@ func (r *KustomizationReconciler) prune(kustomization kustomizev1.Kustomization,
return nil
}
if !prune(kustomization.GetTimeout(),
if output, ok := prune(kustomization.GetTimeout(),
kustomization.GetName(),
kustomization.GetNamespace(),
kustomization.Status.Snapshot,
r.Log,
) {
); !ok {
return fmt.Errorf("pruning failed")
} else {
changeSet := ""
input := strings.Split(output, "\n")
for _, action := range input {
if strings.Contains(action, "deleted") {
changeSet += action + "\n"
}
}
if changeSet != "" {
r.event(kustomization, snapshot.Revision, recorder.EventSeverityInfo, changeSet)
}
}
return nil
}
@ -645,7 +660,11 @@ func (r *KustomizationReconciler) event(kustomization kustomizev1.Kustomization,
}
if r.ExternalEventRecorder != nil {
if err := r.ExternalEventRecorder.Eventf(*objRef, map[string]string{"revision": revision}, severity, severity, msg); err != nil {
var meta map[string]string
if revision != "" {
meta = map[string]string{"revision": revision}
}
if err := r.ExternalEventRecorder.Eventf(*objRef, meta, severity, severity, msg); err != nil {
r.Log.WithValues(
strings.ToLower(kustomization.Kind),
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),

View File

@ -78,9 +78,10 @@ func (gc KustomizationGarbageCollectPredicate) Delete(e event.DeleteEvent) bool
return true
}
func prune(timeout time.Duration, name string, namespace string, snapshot *kustomizev1.Snapshot, log logr.Logger) bool {
func prune(timeout time.Duration, name string, namespace string, snapshot *kustomizev1.Snapshot, log logr.Logger) (string, bool) {
selector := gcSelectors(name, namespace, snapshot.Revision)
ok := true
changeSet := ""
outInfo := ""
outErr := ""
for ns, kinds := range snapshot.NamespacedKinds() {
@ -89,7 +90,7 @@ func prune(timeout time.Duration, name string, namespace string, snapshot *kusto
outErr += " " + err.Error()
ok = false
} else {
outInfo += " " + output
outInfo += " " + output + "\n"
}
}
}
@ -97,6 +98,7 @@ func prune(timeout time.Duration, name string, namespace string, snapshot *kusto
log.Info("Garbage collection for namespaced objects completed",
"kustomization", fmt.Sprintf("%s/%s", namespace, name),
"output", outInfo)
changeSet += outInfo
} else {
log.Error(fmt.Errorf(outErr), "Garbage collection for namespaced objects failed",
"kustomization", fmt.Sprintf("%s/%s", namespace, name))
@ -109,19 +111,20 @@ func prune(timeout time.Duration, name string, namespace string, snapshot *kusto
outErr += " " + err.Error()
ok = false
} else {
outInfo += " " + output
outInfo += " " + output + "\n"
}
}
if outErr == "" {
log.Info("Garbage collection for non-namespaced objects completed",
"kustomization", fmt.Sprintf("%s/%s", namespace, name),
"output", outInfo)
changeSet += outInfo
} else {
log.Error(fmt.Errorf(outErr), "Garbage collection for non-namespaced objects failed",
"kustomization", fmt.Sprintf("%s/%s", namespace, name))
}
return ok
return changeSet, ok
}
func deleteByKind(timeout time.Duration, kind, namespace, selector string) (string, error) {

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 34 KiB

View File

@ -190,22 +190,4 @@ spec:
validate: server
```
Configure alerting for all pipelines in the `kustomize-system` namespace:
```yaml
apiVersion: kustomize.fluxcd.io/v1alpha1
kind: Profile
metadata:
name: default
namespace: kustomize-system
spec:
alert:
type: slack
verbosity: info
address: https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK
username: kustomize-controller
channel: general
kustomizations:
- '*'
```