fluxcd
/
kustomize-controller
mirror of https://github.com/fluxcd/kustomize-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement Slack and Discord alerting

This commit is contained in:
stefanprodan 2020-04-21 14:09:26 +03:00
parent 057096853e
commit 81ff97bc8d
2 changed files with 214 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
controllers/kustomization_controller.go
View File

@ -21,7 +21,6 @@ import (
"errors"
"fmt"
"io/ioutil"
corev1 "k8s.io/api/core/v1"
"os"
"os/exec"
"path"
@ -29,6 +28,7 @@ import (
"time"
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
@ -36,6 +36,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/controller"
kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1"
"github.com/fluxcd/kustomize-controller/internal/alert"
"github.com/fluxcd/kustomize-controller/internal/lockedfile"
sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
)
@ -118,7 +119,9 @@ func (r *KustomizationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
// we can't rely on exponential backoff because it will prolong the execution too much,
// instead we requeue every half a minute.
requeueAfter := 30 * time.Second
log.Error(err, "Dependencies do not meet ready condition, retrying in "+requeueAfter.String())
msg := fmt.Sprintf("Dependencies do not meet ready condition, retrying in %s", requeueAfter.String())
log.Error(err, msg)
r.alert(kustomization, msg, "info")
return ctrl.Result{RequeueAfter: requeueAfter}, nil
}
log.Info("All dependencies area ready, proceeding with apply")
@ -128,6 +131,7 @@ func (r *KustomizationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
syncedKustomization, err := r.sync(*kustomization.DeepCopy(), source)
if err != nil {
log.Error(err, "Kustomization apply failed")
r.alert(kustomization, err.Error(), "error")
}
// update status
@ -292,7 +296,7 @@ func (r *KustomizationReconciler) validate(kustomization kustomizev1.Kustomizati
output, err := command.CombinedOutput()
if err != nil {
if errors.Is(err, context.DeadlineExceeded) {
return err
return fmt.Errorf("validation timeout: %w", err)
}
return fmt.Errorf("validation failed: %s", string(output))
}
@ -314,19 +318,31 @@ func (r *KustomizationReconciler) apply(kustomization kustomizev1.Kustomization,
output, err := command.CombinedOutput()
if err != nil {
if errors.Is(err, context.DeadlineExceeded) {
return err
return fmt.Errorf("apply timeout: %w", err)
}
return fmt.Errorf("apply failed: %s", string(output))
}
resources := r.parseApplyOutput(output)
r.Log.WithValues(
strings.ToLower(kustomization.Kind),
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
).Info(
fmt.Sprintf("Kustomization applied in %s",
time.Now().Sub(start).String()),
"output", r.parseApplyOutput(output),
"output", resources,
)
var diff bool
for _, action := range resources {
if action != "unchanged" {
diff = true
break
}
}
if diff {
r.alert(kustomization, string(output), "info")
}
return nil
}
@ -334,6 +350,7 @@ func (r *KustomizationReconciler) checkHealth(kustomization kustomizev1.Kustomiz
timeout := kustomization.GetTimeout() + (time.Second * 1)
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
var alerts string
for _, check := range kustomization.Spec.HealthChecks {
cmd := fmt.Sprintf("kubectl -n %s rollout status %s %s --timeout=%s",
@ -342,18 +359,25 @@ func (r *KustomizationReconciler) checkHealth(kustomization kustomizev1.Kustomiz
output, err := command.CombinedOutput()
if err != nil {
if errors.Is(err, context.DeadlineExceeded) {
return err
return fmt.Errorf("health check timeout for %s '%s/%s': %w",
check.Kind, check.Namespace, check.Name, err)
}
return fmt.Errorf("health check failed for %s '%s/%s': %s",
check.Kind, check.Namespace, check.Name, string(output))
} else {
msg := fmt.Sprintf("Health check passed for %s '%s/%s'",
check.Kind, check.Namespace, check.Name)
r.Log.WithValues(
strings.ToLower(kustomization.Kind),
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
).Info(fmt.Sprintf("Health check passed for %s '%s/%s'",
check.Kind, check.Namespace, check.Name))
).Info(msg)
alerts += msg + "\n"
}
}
if alerts != "" {
r.alert(kustomization, alerts, "info")
}
return nil
}
@ -409,3 +433,57 @@ func (r *KustomizationReconciler) checkDependencies(kustomization kustomizev1.Ku
return nil
}
func (r *KustomizationReconciler) getProfiles(kustomization kustomizev1.Kustomization) ([]kustomizev1.Profile, error) {
list := make([]kustomizev1.Profile, 0)
var profiles kustomizev1.ProfileList
err := r.List(context.TODO(), &profiles, client.InNamespace(kustomization.GetNamespace()))
if err != nil {
return list, err
}
// filter profiles that match this kustomization taking into account '*' wildcard
for _, profile := range profiles.Items {
for _, name := range profile.Spec.Kustomizations {
if name == kustomization.GetName() || name == "*" {
list = append(list, profile)
break
}
}
}
return list, nil
}
func (r *KustomizationReconciler) alert(kustomization kustomizev1.Kustomization, msg string, verbosity string) {
profiles, err := r.getProfiles(kustomization)
if err != nil {
r.Log.WithValues(
strings.ToLower(kustomization.Kind),
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
).Error(err, "unable to list profiles")
return
}
for _, profile := range profiles {
if settings := profile.Spec.Alert; settings != nil {
provider, err := alert.NewProvider(settings.Type, settings.Address, settings.Username, settings.Channel)
if err != nil {
r.Log.WithValues(
strings.ToLower(kustomization.Kind),
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
).Error(err, "unable to configure alert provider")
continue
}
if settings.Verbosity == verbosity || verbosity == "error" {
err = provider.Post(kustomization.GetName(), kustomization.GetNamespace(), msg, verbosity)
if err != nil {
r.Log.WithValues(
strings.ToLower(kustomization.Kind),
fmt.Sprintf("%s/%s", kustomization.GetNamespace(), kustomization.GetName()),
).Error(err, "unable to send alert")
}
}
}
}
}

128
internal/alert/provider.go Normal file
View File

@ -0,0 +1,128 @@
package alert
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"path"
"strings"
"time"
)
// Provider holds the information needed to post alerts
type Provider struct {
URL string
Username string
Channel string
}
// Payload holds the channel and attachments
type Payload struct {
Channel string `json:"channel"`
Username string `json:"username"`
Text string `json:"text,omitempty"`
Attachments []Attachment `json:"attachments,omitempty"`
}
// Attachment holds the markdown message body
type Attachment struct {
Color string `json:"color"`
AuthorName string `json:"author_name"`
Text string `json:"text"`
MrkdwnIn []string `json:"mrkdwn_in"`
}
// NewProvider validates the URL and returns a provider object
func NewProvider(providerType string, hookURL string, username string, channel string) (*Provider, error) {
hook, err := url.ParseRequestURI(hookURL)
if err != nil {
return nil, fmt.Errorf("invalid hook URL %s", hookURL)
}
if providerType == "discord" {
// https://birdie0.github.io/discord-webhooks-guide/other/slack_formatting.html
if !strings.HasSuffix(hookURL, "/slack") {
hook.Path = path.Join(hook.Path, "slack")
hookURL = hook.String()
}
}
if username == "" {
return nil, errors.New("empty username")
}
if channel == "" {
return nil, errors.New("empty channel")
}
return &Provider{
Channel: channel,
URL: hookURL,
Username: username,
}, nil
}
// Post message to the provider hook URL
func (s *Provider) Post(name string, namespace string, message string, severity string) error {
payload := Payload{
Channel: s.Channel,
Username: s.Username,
}
color := "good"
if severity == "error" {
color = "danger"
}
a := Attachment{
Color: color,
AuthorName: fmt.Sprintf("%s/%s", namespace, name),
Text: message,
MrkdwnIn: []string{"text"},
}
payload.Attachments = []Attachment{a}
err := postMessage(s.URL, payload)
if err != nil {
return fmt.Errorf("postMessage failed: %w", err)
}
return nil
}
func postMessage(address string, payload interface{}) error {
data, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshalling notification payload failed: %w", err)
}
b := bytes.NewBuffer(data)
req, err := http.NewRequest("POST", address, b)
if err != nil {
return fmt.Errorf("http.NewRequest failed: %w", err)
}
req.Header.Set("Content-type", "application/json")
ctx, cancel := context.WithTimeout(req.Context(), 5*time.Second)
defer cancel()
res, err := http.DefaultClient.Do(req.WithContext(ctx))
if err != nil {
return fmt.Errorf("sending notification failed: %w", err)
}
defer res.Body.Close()
statusCode := res.StatusCode
if statusCode != 200 {
body, _ := ioutil.ReadAll(res.Body)
return fmt.Errorf("sending notification failed: %s", string(body))
}
return nil
}