Merge pull request #311 from andrewjjenkins/mirror

Add traffic mirroring for Istio service mesh

artifacts/flagger/crd.yaml

@@ -41,6 +41,10 @@ spec:
       type: string
       JSONPath: .spec.canaryAnalysis.interval
       priority: 1
+    - name: Mirror
+      type: boolean
+      JSONPath: .spec.canaryAnalysis.mirror
+      priority: 1
     - name: StepWeight
       type: string
       JSONPath: .spec.canaryAnalysis.stepWeight
@@ -183,6 +187,9 @@ spec:
                 stepWeight:
                   description: Canary incremental traffic percentage step
                   type: number
+                mirror:
+                  description: Mirror traffic to canary before shifting
+                  type: boolean
                 match:
                   description: A/B testing match conditions
                   anyOf:

charts/flagger/templates/crd.yaml

@@ -42,6 +42,10 @@ spec:
       type: string
       JSONPath: .spec.canaryAnalysis.interval
       priority: 1
+    - name: Mirror
+      type: boolean
+      JSONPath: .spec.canaryAnalysis.mirror
+      priority: 1
     - name: StepWeight
       type: string
       JSONPath: .spec.canaryAnalysis.stepWeight
@@ -184,6 +188,9 @@ spec:
                 stepWeight:
                   description: Canary incremental traffic percentage step
                   type: number
+                mirror:
+                  description: Mirror traffic to canary before shifting
+                  type: boolean
                 match:
                   description: A/B testing match conditions
                   anyOf:

docs/gitbook/faq.md

@@ -102,6 +102,50 @@ The above configuration will run an analysis for five minutes.
 Flagger starts the load test for the canary service (green version) and checks the Prometheus metrics every 30 seconds.
 If the analysis result is positive, Flagger will promote the canary (green version) to primary (blue version).
 
+**When can I use traffic mirroring?**
+
+Traffic Mirroring is a pre-stage in a Canary (progressive traffic shifting) or
+Blue/Green deployment strategy.  Traffic mirroring will copy each incoming
+request, sending one request to the primary and one to the canary service.  The
+response from the primary is sent back to the user.  The response from the canary
+is discarded.  Metrics are collected on both requests so that the deployment will
+only proceed if the canary metrics are healthy.
+
+Mirroring is supported by Istio only.
+
+In Istio, mirrored requests have `-shadow` appended to the `Host` (HTTP) or
+`Authority` (HTTP/2) header; for example requests to `podinfo.test` that are
+mirrored will be reported in telemetry with a destination host
+`podinfo.test-shadow`.
+
+Mirroring must only be used for requests that are **idempotent** or capable of
+being processed twice (once by the primary and once by the canary).  Reads are
+idempotent.  Before using mirroring on requests that may be writes, you should
+consider what will happen if a write is duplicated and handled by the primary
+and canary.
+
+To use mirroring, set `spec.canaryAnalysis.mirror` to `true`.  Example for
+traffic shifting:
+
+```yaml
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+spec:
+  provider: istio
+  canaryAnalysis:
+    interval: 30s
+    mirror: true
+    stepWeight: 20
+    maxWeight: 50
+    metrics:
+    - interval: 29s
+      name: request-success-rate
+      threshold: 99
+    - interval: 29s
+      name: request-duration
+      threshold: 500
+```
+
 ### Kubernetes services
 
 **How is an application exposed inside the cluster?**

kustomize/base/flagger/crd.yaml

@@ -41,6 +41,10 @@ spec:
       type: string
       JSONPath: .spec.canaryAnalysis.interval
       priority: 1
+    - name: Mirror
+      type: boolean
+      JSONPath: .spec.canaryAnalysis.mirror
+      priority: 1
     - name: StepWeight
       type: string
       JSONPath: .spec.canaryAnalysis.stepWeight
@@ -183,6 +187,9 @@ spec:
                 stepWeight:
                   description: Canary incremental traffic percentage step
                   type: number
+                mirror:
+                  description: Mirror traffic to canary before shifting
+                  type: boolean
                 match:
                   description: A/B testing match conditions
                   anyOf:

pkg/apis/flagger/v1alpha3/types.go

@@ -111,6 +111,7 @@ type CanaryAnalysis struct {
 	Interval   string                           `json:"interval"`
 	Threshold  int                              `json:"threshold"`
 	MaxWeight  int                              `json:"maxWeight"`
+	Mirror     bool                             `json:"mirror,omitempty"`
 	StepWeight int                              `json:"stepWeight"`
 	Metrics    []CanaryMetric                   `json:"metrics"`
 	Webhooks   []CanaryWebhook                  `json:"webhooks,omitempty"`

pkg/controller/controller_test.go

@@ -42,11 +42,9 @@ type Mocks struct {
 	router        router.Interface
 }
 
-func SetupMocks(abtest bool) Mocks {
+func SetupMocks(c *v1alpha3.Canary) Mocks {
-	// init canary
+	if c == nil {
-	c := newTestCanary()
+		c = newTestCanary()
-	if abtest {
-		c = newTestCanaryAB()
 	}
 	flaggerClient := fakeFlagger.NewSimpleClientset(c)
 
@@ -269,6 +267,12 @@ func newTestCanary() *v1alpha3.Canary {
 	return cd
 }
 
+func newTestCanaryMirror() *v1alpha3.Canary {
+	cd := newTestCanary()
+	cd.Spec.CanaryAnalysis.Mirror = true
+	return cd
+}
+
 func newTestCanaryAB() *v1alpha3.Canary {
 	cd := &v1alpha3.Canary{
 		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},

pkg/controller/scheduler.go

@@ -155,7 +155,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 
 	// check if virtual service exists
 	// and if it contains weighted destination routes to the primary and canary services
-	primaryWeight, canaryWeight, err := meshRouter.GetRoutes(cd)
+	primaryWeight, canaryWeight, mirrored, err := meshRouter.GetRoutes(cd)
 	if err != nil {
 		c.recordEventWarningf(cd, "%v", err)
 		return
@@ -176,7 +176,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		// route all traffic back to primary
 		primaryWeight = 100
 		canaryWeight = 0
-		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
 			return
 		}
@@ -218,7 +218,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	if cd.Status.Phase == flaggerv1.CanaryPhasePromoting {
 		if provider != "kubernetes" {
 			c.recordEventInfof(cd, "Routing all traffic to primary")
-			if err := meshRouter.SetRoutes(cd, 100, 0); err != nil {
+			if err := meshRouter.SetRoutes(cd, 100, 0, false); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
@@ -275,7 +275,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		// route all traffic back to primary
 		primaryWeight = 100
 		canaryWeight = 0
-		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
 			return
 		}
@@ -302,8 +302,9 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	}
 
 	// check if the canary success rate is above the threshold
-	// skip check if no traffic is routed to canary
+	// skip check if no traffic is routed or mirrored to canary
-	if canaryWeight == 0 && cd.Status.Iterations == 0 {
+	if canaryWeight == 0 && cd.Status.Iterations == 0 &&
+		(cd.Spec.CanaryAnalysis.Mirror == false || mirrored == false) {
 		c.recordEventInfof(cd, "Starting canary analysis for %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
 
 		// run pre-rollout web hooks
@@ -328,7 +329,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	if len(cd.Spec.CanaryAnalysis.Match) > 0 && cd.Spec.CanaryAnalysis.Iterations > 0 {
 		// route traffic to canary and increment iterations
 		if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
-			if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
+			if err := meshRouter.SetRoutes(cd, 0, 100, false); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
@@ -372,6 +373,15 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	if cd.Spec.CanaryAnalysis.Iterations > 0 {
 		// increment iterations
 		if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
+			// If in "mirror" mode, mirror requests during the entire B/G canary test
+			if provider != "kubernetes" &&
+				cd.Spec.CanaryAnalysis.Mirror == true && mirrored == false {
+				if err := meshRouter.SetRoutes(cd, 100, 0, true); err != nil {
+					c.recordEventWarningf(cd, "%v", err)
+				}
+				c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).
+					Infof("Enabling mirroring for Blue/Green")
+			}
 			if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
@@ -390,7 +400,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
 			if provider != "kubernetes" {
 				c.recordEventInfof(cd, "Routing all traffic to canary")
-				if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
+				if err := meshRouter.SetRoutes(cd, 0, 100, false); err != nil {
 					c.recordEventWarningf(cd, "%v", err)
 					return
 				}
@@ -429,16 +439,34 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	if cd.Spec.CanaryAnalysis.StepWeight > 0 {
 		// increase traffic weight
 		if canaryWeight < maxWeight {
-			primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
+			// If in "mirror" mode, do one step of mirroring before shifting traffic to canary.
-			if primaryWeight < 0 {
+			// When mirroring, all requests go to primary and canary, but only responses from
-				primaryWeight = 0
+			// primary go back to the user.
-			}
+			if cd.Spec.CanaryAnalysis.Mirror && canaryWeight == 0 {
-			canaryWeight += cd.Spec.CanaryAnalysis.StepWeight
+				if mirrored == false {
-			if canaryWeight > 100 {
+					mirrored = true
-				canaryWeight = 100
+					primaryWeight = 100
+					canaryWeight = 0
+				} else {
+					mirrored = false
+					primaryWeight = 100 - cd.Spec.CanaryAnalysis.StepWeight
+					canaryWeight = cd.Spec.CanaryAnalysis.StepWeight
+				}
+				c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).
+					Infof("Running mirror step %d/%d/%t", primaryWeight, canaryWeight, mirrored)
+			} else {
+
+				primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
+				if primaryWeight < 0 {
+					primaryWeight = 0
+				}
+				canaryWeight += cd.Spec.CanaryAnalysis.StepWeight
+				if canaryWeight > 100 {
+					canaryWeight = 100
+				}
 			}
 
-			if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+			if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, mirrored); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
@@ -489,7 +517,7 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.
 	// route all traffic to primary
 	primaryWeight = 100
 	canaryWeight = 0
-	if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+	if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
 		c.recordEventWarningf(cd, "%v", err)
 		return false
 	}

pkg/controller/scheduler_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestScheduler_Init(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	_, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
@@ -18,7 +18,7 @@ func TestScheduler_Init(t *testing.T) {
 }
 
 func TestScheduler_NewRevision(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	// update
@@ -42,7 +42,7 @@ func TestScheduler_NewRevision(t *testing.T) {
 }
 
 func TestScheduler_Rollback(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -66,7 +66,7 @@ func TestScheduler_Rollback(t *testing.T) {
 }
 
 func TestScheduler_SkipAnalysis(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -107,7 +107,7 @@ func TestScheduler_SkipAnalysis(t *testing.T) {
 }
 
 func TestScheduler_NewRevisionReset(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -123,7 +123,7 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 	// advance
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryWeight, canaryWeight, err := mocks.router.GetRoutes(mocks.canary)
+	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -136,6 +136,10 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 		t.Errorf("Got canary route %v wanted %v", canaryWeight, 10)
 	}
 
+	if mirrored != false {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, false)
+	}
+
 	// second update
 	dep2.Spec.Template.Spec.ServiceAccountName = "test"
 	_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
@@ -146,7 +150,7 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 	// detect changes
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryWeight, canaryWeight, err = mocks.router.GetRoutes(mocks.canary)
+	primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -158,10 +162,14 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 	if canaryWeight != 0 {
 		t.Errorf("Got canary route %v wanted %v", canaryWeight, 0)
 	}
+
+	if mirrored != false {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, false)
+	}
 }
 
 func TestScheduler_Promotion(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
@@ -201,14 +209,14 @@ func TestScheduler_Promotion(t *testing.T) {
 	// detect configs changes
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryWeight, canaryWeight, err := mocks.router.GetRoutes(mocks.canary)
+	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	primaryWeight = 60
 	canaryWeight = 40
-	err = mocks.router.SetRoutes(mocks.canary, primaryWeight, canaryWeight)
+	err = mocks.router.SetRoutes(mocks.canary, primaryWeight, canaryWeight, mirrored)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -242,7 +250,7 @@ func TestScheduler_Promotion(t *testing.T) {
 	// finalise
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryWeight, canaryWeight, err = mocks.router.GetRoutes(mocks.canary)
+	primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -255,6 +263,10 @@ func TestScheduler_Promotion(t *testing.T) {
 		t.Errorf("Got canary route %v wanted %v", canaryWeight, 0)
 	}
 
+	if mirrored != false {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, false)
+	}
+
 	primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
@@ -307,8 +319,66 @@ func TestScheduler_Promotion(t *testing.T) {
 	}
 }
 
+func TestScheduler_Mirroring(t *testing.T) {
+	mocks := SetupMocks(newTestCanaryMirror())
+	// init
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// update
+	dep2 := newTestDeploymentV2()
+	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// detect pod spec changes
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// advance
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// check if traffic is mirrored to canary
+	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if primaryWeight != 100 {
+		t.Errorf("Got primary route %v wanted %v", primaryWeight, 100)
+	}
+
+	if canaryWeight != 0 {
+		t.Errorf("Got canary route %v wanted %v", canaryWeight, 0)
+	}
+
+	if mirrored != true {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, true)
+	}
+
+	// advance
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// check if traffic is mirrored to canary
+	primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if primaryWeight != 90 {
+		t.Errorf("Got primary route %v wanted %v", primaryWeight, 90)
+	}
+
+	if canaryWeight != 10 {
+		t.Errorf("Got canary route %v wanted %v", canaryWeight, 10)
+	}
+
+	if mirrored != false {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, false)
+	}
+}
+
 func TestScheduler_ABTesting(t *testing.T) {
-	mocks := SetupMocks(true)
+	mocks := SetupMocks(newTestCanaryAB())
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -326,7 +396,7 @@ func TestScheduler_ABTesting(t *testing.T) {
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	// check if traffic is routed to canary
-	primaryWeight, canaryWeight, err := mocks.router.GetRoutes(mocks.canary)
+	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -339,6 +409,10 @@ func TestScheduler_ABTesting(t *testing.T) {
 		t.Errorf("Got canary route %v wanted %v", canaryWeight, 100)
 	}
 
+	if mirrored != false {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, false)
+	}
+
 	cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
@@ -392,7 +466,7 @@ func TestScheduler_ABTesting(t *testing.T) {
 }
 
 func TestScheduler_PortDiscovery(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 
 	// enable port discovery
 	cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})

pkg/router/appmesh.go

@@ -259,6 +259,7 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
 func (ar *AppMeshRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 	err error,
 ) {
 	targetName := canary.Spec.TargetRef.Name
@@ -293,6 +294,8 @@ func (ar *AppMeshRouter) GetRoutes(canary *flaggerv1.Canary) (
 			vsName, targetName, targetName)
 	}
 
+	mirrored = false
+
 	return
 }
 
@@ -301,6 +304,7 @@ func (ar *AppMeshRouter) SetRoutes(
 	canary *flaggerv1.Canary,
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 ) error {
 	targetName := canary.Spec.TargetRef.Name
 	vsName := fmt.Sprintf("%s.%s", targetName, canary.Namespace)

pkg/router/appmesh_test.go

@@ -161,12 +161,12 @@ func TestAppmeshRouter_GetSetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	err = router.SetRoutes(mocks.appmeshCanary, 60, 40)
+	err = router.SetRoutes(mocks.appmeshCanary, 60, 40, false)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.appmeshCanary)
+	p, c, m, err := router.GetRoutes(mocks.appmeshCanary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -178,4 +178,8 @@ func TestAppmeshRouter_GetSetRoutes(t *testing.T) {
 	if c != 40 {
 		t.Errorf("Got canary weight %v wanted %v", c, 40)
 	}
+
+	if m != false {
+		t.Errorf("Got mirror %v wanted %v", m, false)
+	}
 }

pkg/router/gloo.go

@@ -63,11 +63,11 @@ func NewGlooRouterWithClient(ctx context.Context, routingRuleClient gloov1.Upstr
 // Reconcile creates or updates the Istio virtual service
 func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error {
 	// do we have routes already?
-	if _, _, err := gr.GetRoutes(canary); err == nil {
+	if _, _, _, err := gr.GetRoutes(canary); err == nil {
 		// we have routes, no need to do anything else
 		return nil
 	} else if solokiterror.IsNotExist(err) {
-		return gr.SetRoutes(canary, 100, 0)
+		return gr.SetRoutes(canary, 100, 0, false)
 	} else {
 		return err
 	}
@@ -77,6 +77,7 @@ func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error {
 func (gr *GlooRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 	err error,
 ) {
 	targetName := canary.Spec.TargetRef.Name
@@ -101,6 +102,8 @@ func (gr *GlooRouter) GetRoutes(canary *flaggerv1.Canary) (
 			targetName, canary.Namespace, targetName, targetName)
 	}
 
+	mirrored = false
+
 	return
 }
 
@@ -109,6 +112,7 @@ func (gr *GlooRouter) SetRoutes(
 	canary *flaggerv1.Canary,
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 ) error {
 	targetName := canary.Spec.TargetRef.Name
 

pkg/router/gloo_test.go

@@ -68,15 +68,16 @@ func TestGlooRouter_SetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.canary)
+	p, c, m, err := router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	p = 50
 	c = 50
+	m = false
 
-	err = router.SetRoutes(mocks.canary, p, c)
+	err = router.SetRoutes(mocks.canary, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -127,7 +128,7 @@ func TestGlooRouter_GetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.canary)
+	p, c, m, err := router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -139,4 +140,8 @@ func TestGlooRouter_GetRoutes(t *testing.T) {
 	if c != 0 {
 		t.Errorf("Got canary weight %v wanted %v", c, 0)
 	}
+
+	if m != false {
+		t.Errorf("Got mirror %v wanted %v", m, false)
+	}
 }

pkg/router/ingress.go

@@ -106,19 +106,20 @@ func (i *IngressRouter) Reconcile(canary *flaggerv1.Canary) error {
 func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 	err error,
 ) {
 	canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name)
 	canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{})
 	if err != nil {
-		return 0, 0, err
+		return 0, 0, false, err
 	}
 
 	// A/B testing
 	if len(canary.Spec.CanaryAnalysis.Match) > 0 {
 		for k := range canaryIngress.Annotations {
 			if k == i.GetAnnotationWithPrefix("canary-by-cookie") || k == i.GetAnnotationWithPrefix("canary-by-header") {
-				return 0, 100, nil
+				return 0, 100, false, nil
 			}
 		}
 	}
@@ -128,7 +129,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
 		if k == i.GetAnnotationWithPrefix("canary-weight") {
 			val, err := strconv.Atoi(v)
 			if err != nil {
-				return 0, 0, err
+				return 0, 0, false, err
 			}
 
 			canaryWeight = val
@@ -137,6 +138,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
 	}
 
 	primaryWeight = 100 - canaryWeight
+	mirrored = false
 	return
 }
 
@@ -144,6 +146,7 @@ func (i *IngressRouter) SetRoutes(
 	canary *flaggerv1.Canary,
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 ) error {
 	canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name)
 	canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{})

pkg/router/ingress_test.go

@@ -56,15 +56,16 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.ingressCanary)
+	p, c, m, err := router.GetRoutes(mocks.ingressCanary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	p = 50
 	c = 50
+	m = false
 
-	err = router.SetRoutes(mocks.ingressCanary, p, c)
+	err = router.SetRoutes(mocks.ingressCanary, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -93,8 +94,9 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) {
 
 	p = 100
 	c = 0
+	m = false
 
-	err = router.SetRoutes(mocks.ingressCanary, p, c)
+	err = router.SetRoutes(mocks.ingressCanary, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}

pkg/router/istio.go

@@ -101,8 +101,6 @@ func (ir *IstioRouter) reconcileDestinationRule(canary *flaggerv1.Canary, name s
 
 func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 	targetName := canary.Spec.TargetRef.Name
-	primaryName := fmt.Sprintf("%s-primary", targetName)
-	canaryName := fmt.Sprintf("%s-canary", targetName)
 
 	// set hosts and add the ClusterIP service host if it doesn't exists
 	hosts := canary.Spec.Service.Hosts
@@ -133,6 +131,8 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 	}
 
 	// create destinations with primary weight 100% and canary weight 0%
+	primaryName := fmt.Sprintf("%s-primary", targetName)
+	canaryName := fmt.Sprintf("%s-canary", targetName)
 	canaryRoute := []istiov1alpha3.DestinationWeight{
 		makeDestination(canary, primaryName, 100),
 		makeDestination(canary, canaryName, 0),
@@ -210,9 +210,14 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 		return fmt.Errorf("VirtualService %s.%s query error %v", targetName, canary.Namespace, err)
 	}
 
-	// update service but keep the original destination weights
+	// update service but keep the original destination weights and mirror
 	if virtualService != nil {
-		if diff := cmp.Diff(newSpec, virtualService.Spec, cmpopts.IgnoreFields(istiov1alpha3.DestinationWeight{}, "Weight")); diff != "" {
+		if diff := cmp.Diff(
+			newSpec,
+			virtualService.Spec,
+			cmpopts.IgnoreFields(istiov1alpha3.DestinationWeight{}, "Weight"),
+			cmpopts.IgnoreFields(istiov1alpha3.HTTPRoute{}, "Mirror"),
+		); diff != "" {
 			vtClone := virtualService.DeepCopy()
 			vtClone.Spec = newSpec
 
@@ -232,6 +237,7 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 func (ir *IstioRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 	err error,
 ) {
 	targetName := canary.Spec.TargetRef.Name
@@ -264,6 +270,9 @@ func (ir *IstioRouter) GetRoutes(canary *flaggerv1.Canary) (
 			canaryWeight = route.Weight
 		}
 	}
+	if httpRoute.Mirror != nil && httpRoute.Mirror.Host != "" {
+		mirrored = true
+	}
 
 	if primaryWeight == 0 && canaryWeight == 0 {
 		err = fmt.Errorf("VirtualService %s.%s does not contain routes for %s-primary and %s-canary",
@@ -278,6 +287,7 @@ func (ir *IstioRouter) SetRoutes(
 	canary *flaggerv1.Canary,
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 ) error {
 	targetName := canary.Spec.TargetRef.Name
 	primaryName := fmt.Sprintf("%s-primary", targetName)
@@ -310,6 +320,12 @@ func (ir *IstioRouter) SetRoutes(
 		},
 	}
 
+	if mirrored {
+		vsCopy.Spec.Http[0].Mirror = &istiov1alpha3.Destination{
+			Host: canaryName,
+		}
+	}
+
 	// fix routing (A/B testing)
 	if len(canary.Spec.CanaryAnalysis.Match) > 0 {
 		// merge the common routes with the canary ones

pkg/router/istio_test.go

@@ -119,15 +119,16 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.canary)
+	p, c, m, err := router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	p = 50
+	p = 60
-	c = 50
+	c = 40
+	m = false
 
-	err = router.SetRoutes(mocks.canary, p, c)
+	err = router.SetRoutes(mocks.canary, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -137,16 +138,20 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
+	pHost := fmt.Sprintf("%s-primary", mocks.canary.Spec.TargetRef.Name)
+	cHost := fmt.Sprintf("%s-canary", mocks.canary.Spec.TargetRef.Name)
 	pRoute := istiov1alpha3.DestinationWeight{}
 	cRoute := istiov1alpha3.DestinationWeight{}
+	var mirror *istiov1alpha3.Destination
 
 	for _, http := range vs.Spec.Http {
 		for _, route := range http.Route {
-			if route.Destination.Host == fmt.Sprintf("%s-primary", mocks.canary.Spec.TargetRef.Name) {
+			if route.Destination.Host == pHost {
 				pRoute = route
 			}
-			if route.Destination.Host == fmt.Sprintf("%s-canary", mocks.canary.Spec.TargetRef.Name) {
+			if route.Destination.Host == cHost {
 				cRoute = route
+				mirror = http.Mirror
 			}
 		}
 	}
@@ -158,6 +163,51 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
 	if cRoute.Weight != c {
 		t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c)
 	}
+
+	if mirror != nil {
+		t.Errorf("Got mirror %v wanted nil", mirror)
+	}
+
+	mirror = nil
+	p = 100
+	c = 0
+	m = true
+
+	err = router.SetRoutes(mocks.canary, p, c, m)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	vs, err = mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	for _, http := range vs.Spec.Http {
+		for _, route := range http.Route {
+			if route.Destination.Host == pHost {
+				pRoute = route
+			}
+			if route.Destination.Host == cHost {
+				cRoute = route
+				mirror = http.Mirror
+			}
+		}
+	}
+
+	if pRoute.Weight != p {
+		t.Errorf("Got primary weight %v wanted %v", pRoute.Weight, p)
+	}
+
+	if cRoute.Weight != c {
+		t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c)
+	}
+
+	if mirror == nil {
+		t.Errorf("Got mirror nil wanted a mirror")
+	} else if mirror.Host != cHost {
+		t.Errorf("Got mirror host \"%v\" wanted \"%v\"", mirror.Host, cHost)
+	}
 }
 
 func TestIstioRouter_GetRoutes(t *testing.T) {
@@ -174,7 +224,7 @@ func TestIstioRouter_GetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.canary)
+	p, c, m, err := router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -186,6 +236,74 @@ func TestIstioRouter_GetRoutes(t *testing.T) {
 	if c != 0 {
 		t.Errorf("Got canary weight %v wanted %v", c, 0)
 	}
+
+	if m != false {
+		t.Errorf("Got mirror %v wanted %v", m, false)
+	}
+
+	mocks.canary = newMockMirror()
+
+	err = router.Reconcile(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	p, c, m, err = router.GetRoutes(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if p != 100 {
+		t.Errorf("Got primary weight %v wanted %v", p, 100)
+	}
+
+	if c != 0 {
+		t.Errorf("Got canary weight %v wanted %v", c, 0)
+	}
+
+	// A Canary resource with mirror on does not automatically create mirroring
+	// in the virtual server (mirroring is activated as a temporary stage).
+	if m != false {
+		t.Errorf("Got mirror %v wanted %v", m, false)
+	}
+
+	// Adjust vs to activate mirroring.
+	vs, err := mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	cHost := fmt.Sprintf("%s-canary", mocks.canary.Spec.TargetRef.Name)
+	for i, http := range vs.Spec.Http {
+		for _, route := range http.Route {
+			if route.Destination.Host == cHost {
+				vs.Spec.Http[i].Mirror = &istiov1alpha3.Destination{
+					Host: cHost,
+				}
+			}
+		}
+	}
+	_, err = mocks.meshClient.NetworkingV1alpha3().VirtualServices(mocks.canary.Namespace).Update(vs)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	p, c, m, err = router.GetRoutes(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if p != 100 {
+		t.Errorf("Got primary weight %v wanted %v", p, 100)
+	}
+
+	if c != 0 {
+		t.Errorf("Got canary weight %v wanted %v", c, 0)
+	}
+
+	if m != true {
+		t.Errorf("Got mirror %v wanted %v", m, true)
+	}
 }
 
 func TestIstioRouter_HTTPRequestHeaders(t *testing.T) {
@@ -276,8 +394,9 @@ func TestIstioRouter_ABTest(t *testing.T) {
 
 	p := 0
 	c := 100
+	m := false
 
-	err = router.SetRoutes(mocks.abtest, p, c)
+	err = router.SetRoutes(mocks.abtest, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -287,16 +406,20 @@ func TestIstioRouter_ABTest(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
+	pHost := fmt.Sprintf("%s-primary", mocks.abtest.Spec.TargetRef.Name)
+	cHost := fmt.Sprintf("%s-canary", mocks.abtest.Spec.TargetRef.Name)
 	pRoute := istiov1alpha3.DestinationWeight{}
 	cRoute := istiov1alpha3.DestinationWeight{}
+	var mirror *istiov1alpha3.Destination
 
 	for _, http := range vs.Spec.Http {
 		for _, route := range http.Route {
-			if route.Destination.Host == fmt.Sprintf("%s-primary", mocks.abtest.Spec.TargetRef.Name) {
+			if route.Destination.Host == pHost {
 				pRoute = route
 			}
-			if route.Destination.Host == fmt.Sprintf("%s-canary", mocks.abtest.Spec.TargetRef.Name) {
+			if route.Destination.Host == cHost {
 				cRoute = route
+				mirror = http.Mirror
 			}
 		}
 	}
@@ -308,4 +431,8 @@ func TestIstioRouter_ABTest(t *testing.T) {
 	if cRoute.Weight != c {
 		t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c)
 	}
+
+	if mirror != nil {
+		t.Errorf("Got mirror %v wanted nil", mirror)
+	}
 }

pkg/router/nop.go

@@ -12,13 +12,13 @@ func (*NopRouter) Reconcile(canary *flaggerv1.Canary) error {
 	return nil
 }
 
-func (*NopRouter) SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int) error {
+func (*NopRouter) SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int, mirror bool) error {
 	return nil
 }
 
-func (*NopRouter) GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, err error) {
+func (*NopRouter) GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, mirror bool, err error) {
 	if canary.Status.Iterations > 0 {
-		return 0, 100, nil
+		return 0, 100, false, nil
 	}
-	return 100, 0, nil
+	return 100, 0, false, nil
 }

pkg/router/router.go

@@ -4,6 +4,6 @@ import flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 
 type Interface interface {
 	Reconcile(canary *flaggerv1.Canary) error
-	SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int) error
+	SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int, mirrored bool) error
-	GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, err error)
+	GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, mirrored bool, err error)
 }

pkg/router/router_test.go

@@ -137,6 +137,12 @@ func newMockCanary() *v1alpha3.Canary {
 	return cd
 }
 
+func newMockMirror() *v1alpha3.Canary {
+	cd := newMockCanary()
+	cd.Spec.CanaryAnalysis.Mirror = true
+	return cd
+}
+
 func newMockABTest() *v1alpha3.Canary {
 	cd := &v1alpha3.Canary{
 		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},

pkg/router/smi.go

@@ -107,6 +107,7 @@ func (sr *SmiRouter) Reconcile(canary *flaggerv1.Canary) error {
 func (sr *SmiRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 	err error,
 ) {
 	targetName := canary.Spec.TargetRef.Name
@@ -137,6 +138,8 @@ func (sr *SmiRouter) GetRoutes(canary *flaggerv1.Canary) (
 			targetName, canary.Namespace, primaryName, canaryName)
 	}
 
+	mirrored = false
+
 	return
 }
 
@@ -145,6 +148,7 @@ func (sr *SmiRouter) SetRoutes(
 	canary *flaggerv1.Canary,
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 ) error {
 	targetName := canary.Spec.TargetRef.Name
 	canaryName := fmt.Sprintf("%s-canary", targetName)

pkg/router/supergloo.go

@@ -82,11 +82,11 @@ func (sr *SuperglooRouter) Reconcile(canary *flaggerv1.Canary) error {
 	}
 
 	// do we have routes already?
-	if _, _, err := sr.GetRoutes(canary); err == nil {
+	if _, _, _, err := sr.GetRoutes(canary); err == nil {
 		// we have routes, no need to do anything else
 		return nil
 	} else if solokiterror.IsNotExist(err) {
-		return sr.SetRoutes(canary, 100, 0)
+		return sr.SetRoutes(canary, 100, 0, false)
 	} else {
 		return err
 	}
@@ -219,6 +219,7 @@ func (sr *SuperglooRouter) createRule(canary *flaggerv1.Canary, namesuffix strin
 func (sr *SuperglooRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 	err error,
 ) {
 	targetName := canary.Spec.TargetRef.Name
@@ -247,6 +248,8 @@ func (sr *SuperglooRouter) GetRoutes(canary *flaggerv1.Canary) (
 			targetName, canary.Namespace, targetName, targetName)
 	}
 
+	mirrored = false
+
 	return
 }
 
@@ -259,6 +262,7 @@ func (sr *SuperglooRouter) SetRoutes(
 	canary *flaggerv1.Canary,
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 ) error {
 	// upstream name is
 	// in gloo-system

pkg/router/supergloo_test.go

@@ -71,15 +71,16 @@ func TestSuperglooRouter_SetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.canary)
+	p, c, m, err := router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	p = 50
 	c = 50
+	m = false
 
-	err = router.SetRoutes(mocks.canary, p, c)
+	err = router.SetRoutes(mocks.canary, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -134,7 +135,7 @@ func TestSuperglooRouter_GetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.canary)
+	p, c, m, err := router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -146,4 +147,8 @@ func TestSuperglooRouter_GetRoutes(t *testing.T) {
 	if c != 0 {
 		t.Errorf("Got canary weight %v wanted %v", c, 0)
 	}
+
+	if m != false {
+		t.Errorf("Got mirror %v wanted %v", m, false)
+	}
 }