fluxcd
/
flagger
mirror of https://github.com/fluxcd/flagger.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #311 from andrewjjenkins/mirror 

Add traffic mirroring for Istio service mesh
This commit is contained in:
Stefan Prodan 2019-10-05 10:34:25 +03:00 committed by GitHub
parent 0c60cf39f8 a21e53fa31
commit 9a9baadf0e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 430 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
artifacts/flagger/crd.yaml
View File

@ -41,6 +41,10 @@ spec:
type: string
JSONPath: .spec.canaryAnalysis.interval
priority: 1
- name: Mirror
type: boolean
JSONPath: .spec.canaryAnalysis.mirror
priority: 1
- name: StepWeight
type: string
JSONPath: .spec.canaryAnalysis.stepWeight
@ -183,6 +187,9 @@ spec:
stepWeight:
description: Canary incremental traffic percentage step
type: number
mirror:
description: Mirror traffic to canary before shifting
type: boolean
match:
description: A/B testing match conditions
anyOf:

7
charts/flagger/templates/crd.yaml
View File

@ -42,6 +42,10 @@ spec:
type: string
JSONPath: .spec.canaryAnalysis.interval
priority: 1
- name: Mirror
type: boolean
JSONPath: .spec.canaryAnalysis.mirror
priority: 1
- name: StepWeight
type: string
JSONPath: .spec.canaryAnalysis.stepWeight
@ -184,6 +188,9 @@ spec:
stepWeight:
description: Canary incremental traffic percentage step
type: number
mirror:
description: Mirror traffic to canary before shifting
type: boolean
match:
description: A/B testing match conditions
anyOf:

44
docs/gitbook/faq.md
View File

@ -102,6 +102,50 @@ The above configuration will run an analysis for five minutes.
Flagger starts the load test for the canary service (green version) and checks the Prometheus metrics every 30 seconds.
If the analysis result is positive, Flagger will promote the canary (green version) to primary (blue version).
**When can I use traffic mirroring?**
Traffic Mirroring is a pre-stage in a Canary (progressive traffic shifting) or
Blue/Green deployment strategy. Traffic mirroring will copy each incoming
request, sending one request to the primary and one to the canary service. The
response from the primary is sent back to the user. The response from the canary
is discarded. Metrics are collected on both requests so that the deployment will
only proceed if the canary metrics are healthy.
Mirroring is supported by Istio only.
In Istio, mirrored requests have `-shadow` appended to the `Host` (HTTP) or
`Authority` (HTTP/2) header; for example requests to `podinfo.test` that are
mirrored will be reported in telemetry with a destination host
`podinfo.test-shadow`.
Mirroring must only be used for requests that are **idempotent** or capable of
being processed twice (once by the primary and once by the canary). Reads are
idempotent. Before using mirroring on requests that may be writes, you should
consider what will happen if a write is duplicated and handled by the primary
and canary.
To use mirroring, set `spec.canaryAnalysis.mirror` to `true`. Example for
traffic shifting:
```yaml
apiVersion: flagger.app/v1alpha3
kind: Canary
spec:
provider: istio
canaryAnalysis:
interval: 30s
mirror: true
stepWeight: 20
maxWeight: 50
metrics:
- interval: 29s
name: request-success-rate
threshold: 99
- interval: 29s
name: request-duration
threshold: 500
```
### Kubernetes services
**How is an application exposed inside the cluster?**

7
kustomize/base/flagger/crd.yaml
View File

@ -41,6 +41,10 @@ spec:
type: string
JSONPath: .spec.canaryAnalysis.interval
priority: 1
- name: Mirror
type: boolean
JSONPath: .spec.canaryAnalysis.mirror
priority: 1
- name: StepWeight
type: string
JSONPath: .spec.canaryAnalysis.stepWeight
@ -183,6 +187,9 @@ spec:
stepWeight:
description: Canary incremental traffic percentage step
type: number
mirror:
description: Mirror traffic to canary before shifting
type: boolean
match:
description: A/B testing match conditions
anyOf:

1
pkg/apis/flagger/v1alpha3/types.go Executable file → Normal file
View File

@ -111,6 +111,7 @@ type CanaryAnalysis struct {
Interval string `json:"interval"`
Threshold int `json:"threshold"`
MaxWeight int `json:"maxWeight"`
Mirror bool `json:"mirror,omitempty"`
StepWeight int `json:"stepWeight"`
Metrics []CanaryMetric `json:"metrics"`
Webhooks []CanaryWebhook `json:"webhooks,omitempty"`

14
pkg/controller/controller_test.go
View File

@ -42,11 +42,9 @@ type Mocks struct {
router router.Interface
}
func SetupMocks(abtest bool) Mocks {
// init canary
c := newTestCanary()
if abtest {
c = newTestCanaryAB()
func SetupMocks(c *v1alpha3.Canary) Mocks {
if c == nil {
c = newTestCanary()
}
flaggerClient := fakeFlagger.NewSimpleClientset(c)
@ -269,6 +267,12 @@ func newTestCanary() *v1alpha3.Canary {
return cd
}
func newTestCanaryMirror() *v1alpha3.Canary {
cd := newTestCanary()
cd.Spec.CanaryAnalysis.Mirror = true
return cd
}
func newTestCanaryAB() *v1alpha3.Canary {
cd := &v1alpha3.Canary{
TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},

62
pkg/controller/scheduler.go
View File

@ -155,7 +155,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
// check if virtual service exists
// and if it contains weighted destination routes to the primary and canary services
primaryWeight, canaryWeight, err := meshRouter.GetRoutes(cd)
primaryWeight, canaryWeight, mirrored, err := meshRouter.GetRoutes(cd)
if err != nil {
c.recordEventWarningf(cd, "%v", err)
return
@ -176,7 +176,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
// route all traffic back to primary
primaryWeight = 100
canaryWeight = 0
if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
@ -218,7 +218,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
if cd.Status.Phase == flaggerv1.CanaryPhasePromoting {
if provider != "kubernetes" {
c.recordEventInfof(cd, "Routing all traffic to primary")
if err := meshRouter.SetRoutes(cd, 100, 0); err != nil {
if err := meshRouter.SetRoutes(cd, 100, 0, false); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
@ -275,7 +275,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
// route all traffic back to primary
primaryWeight = 100
canaryWeight = 0
if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
@ -302,8 +302,9 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
}
// check if the canary success rate is above the threshold
// skip check if no traffic is routed to canary
if canaryWeight == 0 && cd.Status.Iterations == 0 {
// skip check if no traffic is routed or mirrored to canary
if canaryWeight == 0 && cd.Status.Iterations == 0 &&
(cd.Spec.CanaryAnalysis.Mirror == false || mirrored == false) {
c.recordEventInfof(cd, "Starting canary analysis for %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
// run pre-rollout web hooks
@ -328,7 +329,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
if len(cd.Spec.CanaryAnalysis.Match) > 0 && cd.Spec.CanaryAnalysis.Iterations > 0 {
// route traffic to canary and increment iterations
if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
if err := meshRouter.SetRoutes(cd, 0, 100, false); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
@ -372,6 +373,15 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
if cd.Spec.CanaryAnalysis.Iterations > 0 {
// increment iterations
if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
// If in "mirror" mode, mirror requests during the entire B/G canary test
if provider != "kubernetes" &&
cd.Spec.CanaryAnalysis.Mirror == true && mirrored == false {
if err := meshRouter.SetRoutes(cd, 100, 0, true); err != nil {
c.recordEventWarningf(cd, "%v", err)
}
c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).
Infof("Enabling mirroring for Blue/Green")
}
if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
@ -390,7 +400,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
if provider != "kubernetes" {
c.recordEventInfof(cd, "Routing all traffic to canary")
if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
if err := meshRouter.SetRoutes(cd, 0, 100, false); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
@ -429,16 +439,34 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
if cd.Spec.CanaryAnalysis.StepWeight > 0 {
// increase traffic weight
if canaryWeight < maxWeight {
primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
if primaryWeight < 0 {
primaryWeight = 0
}
canaryWeight += cd.Spec.CanaryAnalysis.StepWeight
if canaryWeight > 100 {
canaryWeight = 100
// If in "mirror" mode, do one step of mirroring before shifting traffic to canary.
// When mirroring, all requests go to primary and canary, but only responses from
// primary go back to the user.
if cd.Spec.CanaryAnalysis.Mirror && canaryWeight == 0 {
if mirrored == false {
mirrored = true
primaryWeight = 100
canaryWeight = 0
} else {
mirrored = false
primaryWeight = 100 - cd.Spec.CanaryAnalysis.StepWeight
canaryWeight = cd.Spec.CanaryAnalysis.StepWeight
}
c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).
Infof("Running mirror step %d/%d/%t", primaryWeight, canaryWeight, mirrored)
} else {
primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
if primaryWeight < 0 {
primaryWeight = 0
}
canaryWeight += cd.Spec.CanaryAnalysis.StepWeight
if canaryWeight > 100 {
canaryWeight = 100
}
}
if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, mirrored); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
@ -489,7 +517,7 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.
// route all traffic to primary
primaryWeight = 100
canaryWeight = 0
if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
c.recordEventWarningf(cd, "%v", err)
return false
}

102
pkg/controller/scheduler_test.go
View File

@ -8,7 +8,7 @@ import (
)
func TestScheduler_Init(t *testing.T) {
mocks := SetupMocks(false)
mocks := SetupMocks(nil)
mocks.ctrl.advanceCanary("podinfo", "default", true)
_, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
@ -18,7 +18,7 @@ func TestScheduler_Init(t *testing.T) {
}
func TestScheduler_NewRevision(t *testing.T) {
mocks := SetupMocks(false)
mocks := SetupMocks(nil)
mocks.ctrl.advanceCanary("podinfo", "default", true)
// update
@ -42,7 +42,7 @@ func TestScheduler_NewRevision(t *testing.T) {
}
func TestScheduler_Rollback(t *testing.T) {
mocks := SetupMocks(false)
mocks := SetupMocks(nil)
// init
mocks.ctrl.advanceCanary("podinfo", "default", true)
@ -66,7 +66,7 @@ func TestScheduler_Rollback(t *testing.T) {
}
func TestScheduler_SkipAnalysis(t *testing.T) {
mocks := SetupMocks(false)
mocks := SetupMocks(nil)
// init
mocks.ctrl.advanceCanary("podinfo", "default", true)
@ -107,7 +107,7 @@ func TestScheduler_SkipAnalysis(t *testing.T) {
}
func TestScheduler_NewRevisionReset(t *testing.T) {
mocks := SetupMocks(false)
mocks := SetupMocks(nil)
// init
mocks.ctrl.advanceCanary("podinfo", "default", true)
@ -123,7 +123,7 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
// advance
mocks.ctrl.advanceCanary("podinfo", "default", true)
primaryWeight, canaryWeight, err := mocks.router.GetRoutes(mocks.canary)
primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
@ -136,6 +136,10 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
t.Errorf("Got canary route %v wanted %v", canaryWeight, 10)
}
if mirrored != false {
t.Errorf("Got mirrored %v wanted %v", mirrored, false)
}
// second update
dep2.Spec.Template.Spec.ServiceAccountName = "test"
_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
@ -146,7 +150,7 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
// detect changes
mocks.ctrl.advanceCanary("podinfo", "default", true)
primaryWeight, canaryWeight, err = mocks.router.GetRoutes(mocks.canary)
primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
@ -158,10 +162,14 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
if canaryWeight != 0 {
t.Errorf("Got canary route %v wanted %v", canaryWeight, 0)
}
if mirrored != false {
t.Errorf("Got mirrored %v wanted %v", mirrored, false)
}
}
func TestScheduler_Promotion(t *testing.T) {
mocks := SetupMocks(false)
mocks := SetupMocks(nil)
// init
mocks.ctrl.advanceCanary("podinfo", "default", true)
@ -201,14 +209,14 @@ func TestScheduler_Promotion(t *testing.T) {
// detect configs changes
mocks.ctrl.advanceCanary("podinfo", "default", true)
primaryWeight, canaryWeight, err := mocks.router.GetRoutes(mocks.canary)
primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
primaryWeight = 60
canaryWeight = 40
err = mocks.router.SetRoutes(mocks.canary, primaryWeight, canaryWeight)
err = mocks.router.SetRoutes(mocks.canary, primaryWeight, canaryWeight, mirrored)
if err != nil {
t.Fatal(err.Error())
}
@ -242,7 +250,7 @@ func TestScheduler_Promotion(t *testing.T) {
// finalise
mocks.ctrl.advanceCanary("podinfo", "default", true)
primaryWeight, canaryWeight, err = mocks.router.GetRoutes(mocks.canary)
primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
@ -255,6 +263,10 @@ func TestScheduler_Promotion(t *testing.T) {
t.Errorf("Got canary route %v wanted %v", canaryWeight, 0)
}
if mirrored != false {
t.Errorf("Got mirrored %v wanted %v", mirrored, false)
}
primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
if err != nil {
t.Fatal(err.Error())
@ -307,8 +319,66 @@ func TestScheduler_Promotion(t *testing.T) {
}
}
func TestScheduler_Mirroring(t *testing.T) {
mocks := SetupMocks(newTestCanaryMirror())
// init
mocks.ctrl.advanceCanary("podinfo", "default", true)
// update
dep2 := newTestDeploymentV2()
_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
if err != nil {
t.Fatal(err.Error())
}
// detect pod spec changes
mocks.ctrl.advanceCanary("podinfo", "default", true)
// advance
mocks.ctrl.advanceCanary("podinfo", "default", true)
// check if traffic is mirrored to canary
primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
if primaryWeight != 100 {
t.Errorf("Got primary route %v wanted %v", primaryWeight, 100)
}
if canaryWeight != 0 {
t.Errorf("Got canary route %v wanted %v", canaryWeight, 0)
}
if mirrored != true {
t.Errorf("Got mirrored %v wanted %v", mirrored, true)
}
// advance
mocks.ctrl.advanceCanary("podinfo", "default", true)
// check if traffic is mirrored to canary
primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
if primaryWeight != 90 {
t.Errorf("Got primary route %v wanted %v", primaryWeight, 90)
}
if canaryWeight != 10 {
t.Errorf("Got canary route %v wanted %v", canaryWeight, 10)
}
if mirrored != false {
t.Errorf("Got mirrored %v wanted %v", mirrored, false)
}
}
func TestScheduler_ABTesting(t *testing.T) {
mocks := SetupMocks(true)
mocks := SetupMocks(newTestCanaryAB())
// init
mocks.ctrl.advanceCanary("podinfo", "default", true)
@ -326,7 +396,7 @@ func TestScheduler_ABTesting(t *testing.T) {
mocks.ctrl.advanceCanary("podinfo", "default", true)
// check if traffic is routed to canary
primaryWeight, canaryWeight, err := mocks.router.GetRoutes(mocks.canary)
primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
@ -339,6 +409,10 @@ func TestScheduler_ABTesting(t *testing.T) {
t.Errorf("Got canary route %v wanted %v", canaryWeight, 100)
}
if mirrored != false {
t.Errorf("Got mirrored %v wanted %v", mirrored, false)
}
cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
if err != nil {
t.Fatal(err.Error())
@ -392,7 +466,7 @@ func TestScheduler_ABTesting(t *testing.T) {
}
func TestScheduler_PortDiscovery(t *testing.T) {
mocks := SetupMocks(false)
mocks := SetupMocks(nil)
// enable port discovery
cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})

4
pkg/router/appmesh.go
View File

@ -259,6 +259,7 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
func (ar *AppMeshRouter) GetRoutes(canary *flaggerv1.Canary) (
primaryWeight int,
canaryWeight int,
mirrored bool,
err error,
) {
targetName := canary.Spec.TargetRef.Name
@ -293,6 +294,8 @@ func (ar *AppMeshRouter) GetRoutes(canary *flaggerv1.Canary) (
vsName, targetName, targetName)
}
mirrored = false
return
}
@ -301,6 +304,7 @@ func (ar *AppMeshRouter) SetRoutes(
canary *flaggerv1.Canary,
primaryWeight int,
canaryWeight int,
mirrored bool,
) error {
targetName := canary.Spec.TargetRef.Name
vsName := fmt.Sprintf("%s.%s", targetName, canary.Namespace)

8
pkg/router/appmesh_test.go
View File

@ -161,12 +161,12 @@ func TestAppmeshRouter_GetSetRoutes(t *testing.T) {
t.Fatal(err.Error())
}
err = router.SetRoutes(mocks.appmeshCanary, 60, 40)
err = router.SetRoutes(mocks.appmeshCanary, 60, 40, false)
if err != nil {
t.Fatal(err.Error())
}
p, c, err := router.GetRoutes(mocks.appmeshCanary)
p, c, m, err := router.GetRoutes(mocks.appmeshCanary)
if err != nil {
t.Fatal(err.Error())
}
@ -178,4 +178,8 @@ func TestAppmeshRouter_GetSetRoutes(t *testing.T) {
if c != 40 {
t.Errorf("Got canary weight %v wanted %v", c, 40)
}
if m != false {
t.Errorf("Got mirror %v wanted %v", m, false)
}
}

8
pkg/router/gloo.go
View File

@ -63,11 +63,11 @@ func NewGlooRouterWithClient(ctx context.Context, routingRuleClient gloov1.Upstr
// Reconcile creates or updates the Istio virtual service
func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error {
// do we have routes already?
if _, _, err := gr.GetRoutes(canary); err == nil {
if _, _, _, err := gr.GetRoutes(canary); err == nil {
// we have routes, no need to do anything else
return nil
} else if solokiterror.IsNotExist(err) {
return gr.SetRoutes(canary, 100, 0)
return gr.SetRoutes(canary, 100, 0, false)
} else {
return err
}
@ -77,6 +77,7 @@ func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error {
func (gr *GlooRouter) GetRoutes(canary *flaggerv1.Canary) (
primaryWeight int,
canaryWeight int,
mirrored bool,
err error,
) {
targetName := canary.Spec.TargetRef.Name
@ -101,6 +102,8 @@ func (gr *GlooRouter) GetRoutes(canary *flaggerv1.Canary) (
targetName, canary.Namespace, targetName, targetName)
}
mirrored = false
return
}
@ -109,6 +112,7 @@ func (gr *GlooRouter) SetRoutes(
canary *flaggerv1.Canary,
primaryWeight int,
canaryWeight int,
mirrored bool,
) error {
targetName := canary.Spec.TargetRef.Name

11
pkg/router/gloo_test.go
View File

@ -68,15 +68,16 @@ func TestGlooRouter_SetRoutes(t *testing.T) {
t.Fatal(err.Error())
}
p, c, err := router.GetRoutes(mocks.canary)
p, c, m, err := router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
p = 50
c = 50
m = false
err = router.SetRoutes(mocks.canary, p, c)
err = router.SetRoutes(mocks.canary, p, c, m)
if err != nil {
t.Fatal(err.Error())
}
@ -127,7 +128,7 @@ func TestGlooRouter_GetRoutes(t *testing.T) {
t.Fatal(err.Error())
}
p, c, err := router.GetRoutes(mocks.canary)
p, c, m, err := router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
@ -139,4 +140,8 @@ func TestGlooRouter_GetRoutes(t *testing.T) {
if c != 0 {
t.Errorf("Got canary weight %v wanted %v", c, 0)
}
if m != false {
t.Errorf("Got mirror %v wanted %v", m, false)
}
}

9
pkg/router/ingress.go
View File

@ -106,19 +106,20 @@ func (i *IngressRouter) Reconcile(canary *flaggerv1.Canary) error {
func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
primaryWeight int,
canaryWeight int,
mirrored bool,
err error,
) {
canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name)
canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{})
if err != nil {
return 0, 0, err
return 0, 0, false, err
}
// A/B testing
if len(canary.Spec.CanaryAnalysis.Match) > 0 {
for k := range canaryIngress.Annotations {
if k == i.GetAnnotationWithPrefix("canary-by-cookie") || k == i.GetAnnotationWithPrefix("canary-by-header") {
return 0, 100, nil
return 0, 100, false, nil
}
}
}
@ -128,7 +129,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
if k == i.GetAnnotationWithPrefix("canary-weight") {
val, err := strconv.Atoi(v)
if err != nil {
return 0, 0, err
return 0, 0, false, err
}
canaryWeight = val
@ -137,6 +138,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
}
primaryWeight = 100 - canaryWeight
mirrored = false
return
}
@ -144,6 +146,7 @@ func (i *IngressRouter) SetRoutes(
canary *flaggerv1.Canary,
primaryWeight int,
canaryWeight int,
mirrored bool,
) error {
canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name)
canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{})

8
pkg/router/ingress_test.go
View File

@ -56,15 +56,16 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) {
t.Fatal(err.Error())
}
p, c, err := router.GetRoutes(mocks.ingressCanary)
p, c, m, err := router.GetRoutes(mocks.ingressCanary)
if err != nil {
t.Fatal(err.Error())
}
p = 50
c = 50
m = false
err = router.SetRoutes(mocks.ingressCanary, p, c)
err = router.SetRoutes(mocks.ingressCanary, p, c, m)
if err != nil {
t.Fatal(err.Error())
}
@ -93,8 +94,9 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) {
p = 100
c = 0
m = false
err = router.SetRoutes(mocks.ingressCanary, p, c)
err = router.SetRoutes(mocks.ingressCanary, p, c, m)
if err != nil {
t.Fatal(err.Error())
}

24
pkg/router/istio.go
View File

@ -101,8 +101,6 @@ func (ir *IstioRouter) reconcileDestinationRule(canary *flaggerv1.Canary, name s
func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
targetName := canary.Spec.TargetRef.Name
primaryName := fmt.Sprintf("%s-primary", targetName)
canaryName := fmt.Sprintf("%s-canary", targetName)
// set hosts and add the ClusterIP service host if it doesn't exists
hosts := canary.Spec.Service.Hosts
@ -133,6 +131,8 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
}
// create destinations with primary weight 100% and canary weight 0%
primaryName := fmt.Sprintf("%s-primary", targetName)
canaryName := fmt.Sprintf("%s-canary", targetName)
canaryRoute := []istiov1alpha3.DestinationWeight{
makeDestination(canary, primaryName, 100),
makeDestination(canary, canaryName, 0),
@ -210,9 +210,14 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
return fmt.Errorf("VirtualService %s.%s query error %v", targetName, canary.Namespace, err)
}
// update service but keep the original destination weights
// update service but keep the original destination weights and mirror
if virtualService != nil {
if diff := cmp.Diff(newSpec, virtualService.Spec, cmpopts.IgnoreFields(istiov1alpha3.DestinationWeight{}, "Weight")); diff != "" {
if diff := cmp.Diff(
newSpec,
virtualService.Spec,
cmpopts.IgnoreFields(istiov1alpha3.DestinationWeight{}, "Weight"),
cmpopts.IgnoreFields(istiov1alpha3.HTTPRoute{}, "Mirror"),
); diff != "" {
vtClone := virtualService.DeepCopy()
vtClone.Spec = newSpec
@ -232,6 +237,7 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
func (ir *IstioRouter) GetRoutes(canary *flaggerv1.Canary) (
primaryWeight int,
canaryWeight int,
mirrored bool,
err error,
) {
targetName := canary.Spec.TargetRef.Name
@ -264,6 +270,9 @@ func (ir *IstioRouter) GetRoutes(canary *flaggerv1.Canary) (
canaryWeight = route.Weight
}
}
if httpRoute.Mirror != nil && httpRoute.Mirror.Host != "" {
mirrored = true
}
if primaryWeight == 0 && canaryWeight == 0 {
err = fmt.Errorf("VirtualService %s.%s does not contain routes for %s-primary and %s-canary",
@ -278,6 +287,7 @@ func (ir *IstioRouter) SetRoutes(
canary *flaggerv1.Canary,
primaryWeight int,
canaryWeight int,
mirrored bool,
) error {
targetName := canary.Spec.TargetRef.Name
primaryName := fmt.Sprintf("%s-primary", targetName)
@ -310,6 +320,12 @@ func (ir *IstioRouter) SetRoutes(
},
}
if mirrored {
vsCopy.Spec.Http[0].Mirror = &istiov1alpha3.Destination{
Host: canaryName,
}
}
// fix routing (A/B testing)
if len(canary.Spec.CanaryAnalysis.Match) > 0 {
// merge the common routes with the canary ones

147
pkg/router/istio_test.go
View File

@ -119,15 +119,16 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
t.Fatal(err.Error())
}
p, c, err := router.GetRoutes(mocks.canary)
p, c, m, err := router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
p = 50
c = 50
p = 60
c = 40
m = false
err = router.SetRoutes(mocks.canary, p, c)
err = router.SetRoutes(mocks.canary, p, c, m)
if err != nil {
t.Fatal(err.Error())
}
@ -137,16 +138,20 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
t.Fatal(err.Error())
}
pHost := fmt.Sprintf("%s-primary", mocks.canary.Spec.TargetRef.Name)
cHost := fmt.Sprintf("%s-canary", mocks.canary.Spec.TargetRef.Name)
pRoute := istiov1alpha3.DestinationWeight{}
cRoute := istiov1alpha3.DestinationWeight{}
var mirror *istiov1alpha3.Destination
for _, http := range vs.Spec.Http {
for _, route := range http.Route {
if route.Destination.Host == fmt.Sprintf("%s-primary", mocks.canary.Spec.TargetRef.Name) {
if route.Destination.Host == pHost {
pRoute = route
}
if route.Destination.Host == fmt.Sprintf("%s-canary", mocks.canary.Spec.TargetRef.Name) {
if route.Destination.Host == cHost {
cRoute = route
mirror = http.Mirror
}
}
}
@ -158,6 +163,51 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
if cRoute.Weight != c {
t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c)
}
if mirror != nil {
t.Errorf("Got mirror %v wanted nil", mirror)
}
mirror = nil
p = 100
c = 0
m = true
err = router.SetRoutes(mocks.canary, p, c, m)
if err != nil {
t.Fatal(err.Error())
}
vs, err = mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get("podinfo", metav1.GetOptions{})
if err != nil {
t.Fatal(err.Error())
}
for _, http := range vs.Spec.Http {
for _, route := range http.Route {
if route.Destination.Host == pHost {
pRoute = route
}
if route.Destination.Host == cHost {
cRoute = route
mirror = http.Mirror
}
}
}
if pRoute.Weight != p {
t.Errorf("Got primary weight %v wanted %v", pRoute.Weight, p)
}
if cRoute.Weight != c {
t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c)
}
if mirror == nil {
t.Errorf("Got mirror nil wanted a mirror")
} else if mirror.Host != cHost {
t.Errorf("Got mirror host \"%v\" wanted \"%v\"", mirror.Host, cHost)
}
}
func TestIstioRouter_GetRoutes(t *testing.T) {
@ -174,7 +224,7 @@ func TestIstioRouter_GetRoutes(t *testing.T) {
t.Fatal(err.Error())
}
p, c, err := router.GetRoutes(mocks.canary)
p, c, m, err := router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
@ -186,6 +236,74 @@ func TestIstioRouter_GetRoutes(t *testing.T) {
if c != 0 {
t.Errorf("Got canary weight %v wanted %v", c, 0)
}
if m != false {
t.Errorf("Got mirror %v wanted %v", m, false)
}
mocks.canary = newMockMirror()
err = router.Reconcile(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
p, c, m, err = router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
if p != 100 {
t.Errorf("Got primary weight %v wanted %v", p, 100)
}
if c != 0 {
t.Errorf("Got canary weight %v wanted %v", c, 0)
}
// A Canary resource with mirror on does not automatically create mirroring
// in the virtual server (mirroring is activated as a temporary stage).
if m != false {
t.Errorf("Got mirror %v wanted %v", m, false)
}
// Adjust vs to activate mirroring.
vs, err := mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get("podinfo", metav1.GetOptions{})
if err != nil {
t.Fatal(err.Error())
}
cHost := fmt.Sprintf("%s-canary", mocks.canary.Spec.TargetRef.Name)
for i, http := range vs.Spec.Http {
for _, route := range http.Route {
if route.Destination.Host == cHost {
vs.Spec.Http[i].Mirror = &istiov1alpha3.Destination{
Host: cHost,
}
}
}
}
_, err = mocks.meshClient.NetworkingV1alpha3().VirtualServices(mocks.canary.Namespace).Update(vs)
if err != nil {
t.Fatal(err.Error())
}
p, c, m, err = router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
if p != 100 {
t.Errorf("Got primary weight %v wanted %v", p, 100)
}
if c != 0 {
t.Errorf("Got canary weight %v wanted %v", c, 0)
}
if m != true {
t.Errorf("Got mirror %v wanted %v", m, true)
}
}
func TestIstioRouter_HTTPRequestHeaders(t *testing.T) {
@ -276,8 +394,9 @@ func TestIstioRouter_ABTest(t *testing.T) {
p := 0
c := 100
m := false
err = router.SetRoutes(mocks.abtest, p, c)
err = router.SetRoutes(mocks.abtest, p, c, m)
if err != nil {
t.Fatal(err.Error())
}
@ -287,16 +406,20 @@ func TestIstioRouter_ABTest(t *testing.T) {
t.Fatal(err.Error())
}
pHost := fmt.Sprintf("%s-primary", mocks.abtest.Spec.TargetRef.Name)
cHost := fmt.Sprintf("%s-canary", mocks.abtest.Spec.TargetRef.Name)
pRoute := istiov1alpha3.DestinationWeight{}
cRoute := istiov1alpha3.DestinationWeight{}
var mirror *istiov1alpha3.Destination
for _, http := range vs.Spec.Http {
for _, route := range http.Route {
if route.Destination.Host == fmt.Sprintf("%s-primary", mocks.abtest.Spec.TargetRef.Name) {
if route.Destination.Host == pHost {
pRoute = route
}
if route.Destination.Host == fmt.Sprintf("%s-canary", mocks.abtest.Spec.TargetRef.Name) {
if route.Destination.Host == cHost {
cRoute = route
mirror = http.Mirror
}
}
}
@ -308,4 +431,8 @@ func TestIstioRouter_ABTest(t *testing.T) {
if cRoute.Weight != c {
t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c)
}
if mirror != nil {
t.Errorf("Got mirror %v wanted nil", mirror)
}
}

8
pkg/router/nop.go
View File

@ -12,13 +12,13 @@ func (*NopRouter) Reconcile(canary *flaggerv1.Canary) error {
return nil
}
func (*NopRouter) SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int) error {
func (*NopRouter) SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int, mirror bool) error {
return nil
}
func (*NopRouter) GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, err error) {
func (*NopRouter) GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, mirror bool, err error) {
if canary.Status.Iterations > 0 {
return 0, 100, nil
return 0, 100, false, nil
}
return 100, 0, nil
return 100, 0, false, nil
}

4
pkg/router/router.go
View File

@ -4,6 +4,6 @@ import flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
type Interface interface {
Reconcile(canary *flaggerv1.Canary) error
SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int) error
GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, err error)
SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int, mirrored bool) error
GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, mirrored bool, err error)
}

6
pkg/router/router_test.go
View File

@ -137,6 +137,12 @@ func newMockCanary() *v1alpha3.Canary {
return cd
}
func newMockMirror() *v1alpha3.Canary {
cd := newMockCanary()
cd.Spec.CanaryAnalysis.Mirror = true
return cd
}
func newMockABTest() *v1alpha3.Canary {
cd := &v1alpha3.Canary{
TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},

4
pkg/router/smi.go
View File

@ -107,6 +107,7 @@ func (sr *SmiRouter) Reconcile(canary *flaggerv1.Canary) error {
func (sr *SmiRouter) GetRoutes(canary *flaggerv1.Canary) (
primaryWeight int,
canaryWeight int,
mirrored bool,
err error,
) {
targetName := canary.Spec.TargetRef.Name
@ -137,6 +138,8 @@ func (sr *SmiRouter) GetRoutes(canary *flaggerv1.Canary) (
targetName, canary.Namespace, primaryName, canaryName)
}
mirrored = false
return
}
@ -145,6 +148,7 @@ func (sr *SmiRouter) SetRoutes(
canary *flaggerv1.Canary,
primaryWeight int,
canaryWeight int,
mirrored bool,
) error {
targetName := canary.Spec.TargetRef.Name
canaryName := fmt.Sprintf("%s-canary", targetName)

8
pkg/router/supergloo.go
View File

@ -82,11 +82,11 @@ func (sr *SuperglooRouter) Reconcile(canary *flaggerv1.Canary) error {
}
// do we have routes already?
if _, _, err := sr.GetRoutes(canary); err == nil {
if _, _, _, err := sr.GetRoutes(canary); err == nil {
// we have routes, no need to do anything else
return nil
} else if solokiterror.IsNotExist(err) {
return sr.SetRoutes(canary, 100, 0)
return sr.SetRoutes(canary, 100, 0, false)
} else {
return err
}
@ -219,6 +219,7 @@ func (sr *SuperglooRouter) createRule(canary *flaggerv1.Canary, namesuffix strin
func (sr *SuperglooRouter) GetRoutes(canary *flaggerv1.Canary) (
primaryWeight int,
canaryWeight int,
mirrored bool,
err error,
) {
targetName := canary.Spec.TargetRef.Name
@ -247,6 +248,8 @@ func (sr *SuperglooRouter) GetRoutes(canary *flaggerv1.Canary) (
targetName, canary.Namespace, targetName, targetName)
}
mirrored = false
return
}
@ -259,6 +262,7 @@ func (sr *SuperglooRouter) SetRoutes(
canary *flaggerv1.Canary,
primaryWeight int,
canaryWeight int,
mirrored bool,
) error {
// upstream name is
// in gloo-system

11
pkg/router/supergloo_test.go
View File

@ -71,15 +71,16 @@ func TestSuperglooRouter_SetRoutes(t *testing.T) {
t.Fatal(err.Error())
}
p, c, err := router.GetRoutes(mocks.canary)
p, c, m, err := router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
p = 50
c = 50
m = false
err = router.SetRoutes(mocks.canary, p, c)
err = router.SetRoutes(mocks.canary, p, c, m)
if err != nil {
t.Fatal(err.Error())
}
@ -134,7 +135,7 @@ func TestSuperglooRouter_GetRoutes(t *testing.T) {
t.Fatal(err.Error())
}
p, c, err := router.GetRoutes(mocks.canary)
p, c, m, err := router.GetRoutes(mocks.canary)
if err != nil {
t.Fatal(err.Error())
}
@ -146,4 +147,8 @@ func TestSuperglooRouter_GetRoutes(t *testing.T) {
if c != 0 {
t.Errorf("Got canary weight %v wanted %v", c, 0)
}
if m != false {
t.Errorf("Got mirror %v wanted %v", m, false)
}
}