Implement canary external check

- do a HTTP POST for each webhook registered in the canary analysis
- increment the failed checks counter if a webhook returns a non-2xx status code and log the error and the response body if exists

pkg/apis/flagger/v1alpha2/types.go

@@ -119,6 +119,13 @@ type CanaryWebhook struct {
 	Metadata *map[string]string `json:"metadata,omitempty"`
 }
 
+// CanaryWebhookPayload holds the deployment info and metadata sent to webhooks
+type CanaryWebhookPayload struct {
+	Name      string             `json:"name"`
+	Namespace string             `json:"namespace"`
+	Metadata  *map[string]string `json:"metadata,omitempty"`
+}
+
 // GetProgressDeadlineSeconds returns the progress deadline (default 600s)
 func (c *Canary) GetProgressDeadlineSeconds() int {
 	if c.Spec.ProgressDeadlineSeconds != nil {

pkg/apis/flagger/v1alpha2/zz_generated.deepcopy.go

@@ -223,3 +223,30 @@ func (in *CanaryWebhook) DeepCopy() *CanaryWebhook {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CanaryWebhookPayload) DeepCopyInto(out *CanaryWebhookPayload) {
+	*out = *in
+	if in.Metadata != nil {
+		in, out := &in.Metadata, &out.Metadata
+		*out = new(map[string]string)
+		if **in != nil {
+			in, out := *in, *out
+			*out = make(map[string]string, len(*in))
+			for key, val := range *in {
+				(*out)[key] = val
+			}
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryWebhookPayload.
+func (in *CanaryWebhookPayload) DeepCopy() *CanaryWebhookPayload {
+	if in == nil {
+		return nil
+	}
+	out := new(CanaryWebhookPayload)
+	in.DeepCopyInto(out)
+	return out
+}

pkg/controller/scheduler.go

@@ -139,7 +139,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
 	if canaryRoute.Weight == 0 {
 		c.recordEventInfof(cd, "Starting canary deployment for %s.%s", cd.Name, cd.Namespace)
 	} else {
-		if ok := c.checkCanaryMetrics(cd); !ok {
+		if ok := c.analyseCanary(cd); !ok {
 			if err := c.deployer.SetFailedChecks(cd, cd.Status.FailedChecks+1); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
@@ -242,7 +242,8 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, deployer CanaryDepl
 	return false
 }
 
-func (c *Controller) checkCanaryMetrics(r *flaggerv1.Canary) bool {
+func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
+	// run metrics checks
 	for _, metric := range r.Spec.CanaryAnalysis.Metrics {
 		if metric.Name == "istio_requests_total" {
 			val, err := c.observer.GetDeploymentCounter(r.Spec.TargetRef.Name, r.Namespace, metric.Name, metric.Interval)
@@ -272,5 +273,15 @@ func (c *Controller) checkCanaryMetrics(r *flaggerv1.Canary) bool {
 		}
 	}
 
+	// run external checks
+	for _, webhook := range r.Spec.CanaryAnalysis.Webhooks {
+		err := CallWebhook(r.Name, r.Namespace, webhook)
+		if err != nil {
+			c.recordEventWarningf(r, "Halt %s.%s advancement external check %s failed %v",
+				r.Name, r.Namespace, webhook.Name, err)
+			return false
+		}
+	}
+
 	return true
 }

pkg/controller/webhook.go

@@ -0,0 +1,71 @@
+package controller
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	flaggerv1 "github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha2"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// CallWebhook does a HTTP POST to an external service and
+// returns an error if the response status code is non-2xx
+func CallWebhook(name string, namepace string, w flaggerv1.CanaryWebhook) error {
+
+	payload := flaggerv1.CanaryWebhookPayload{
+		Name:      name,
+		Namespace: namepace,
+		Metadata:  w.Metadata,
+	}
+
+	payloadBin, err := json.Marshal(payload)
+	if err != nil {
+		return err
+	}
+
+	hook, err := url.Parse(w.URL)
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("POST", hook.String(), bytes.NewBuffer(payloadBin))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	if len(w.Timeout) < 2 {
+		w.Timeout = "10s"
+	}
+
+	timeout, err := time.ParseDuration(w.Timeout)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(req.Context(), timeout)
+	defer cancel()
+
+	r, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer r.Body.Close()
+
+	b, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return fmt.Errorf("error reading body: %s", err.Error())
+	}
+
+	if r.StatusCode > 202 {
+		return errors.New(string(b))
+	}
+
+	return nil
+}