Add traffic weight to canary status

- show current weight on kubectl get canaries and kubectl get all
This commit is contained in:
stefanprodan 2019-01-16 16:29:59 +02:00
parent acdd2c46d5
commit 1ef310f00d
8 changed files with 81 additions and 26 deletions

View File

@ -242,15 +242,16 @@ kubectl -n test set image deployment/podinfo \
podinfod=quay.io/stefanprodan/podinfo:1.2.1 podinfod=quay.io/stefanprodan/podinfo:1.2.1
``` ```
Flagger detects that the deployment revision changed and starts a new rollout: Flagger detects that the deployment revision changed and starts a new canary analysis:
``` ```
kubectl -n test describe canary/podinfo kubectl -n test describe canary/podinfo
Status: Status:
Canary Revision: 19871136 Canary Weight: 0
Failed Checks: 0 Failed Checks: 0
State: finished Last Transition Time: 2019-01-16T13:47:16Z
Phase: Succeeded
Events: Events:
Type Reason Age From Message Type Reason Age From Message
---- ------ ---- ---- ------- ---- ------ ---- ---- -------
@ -272,6 +273,15 @@ Events:
Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test
``` ```
You can monitor all canaries with:
```bash
watch kubectl get canaries --all-namespaces
NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME
test podinfo Progressing 5 2019-01-16T14:05:07Z
```
During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout. During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout.
Create a tester pod and exec into it: Create a tester pod and exec into it:
@ -300,9 +310,10 @@ the canary is scaled to zero and the rollout is marked as failed.
kubectl -n test describe canary/podinfo kubectl -n test describe canary/podinfo
Status: Status:
Canary Revision: 16695041 Canary Weight: 0
Failed Checks: 10 Failed Checks: 10
State: failed Last Transition Time: 2019-01-16T13:47:16Z
Phase: Failed
Events: Events:
Type Reason Age From Message Type Reason Age From Message
---- ------ ---- ---- ------- ---- ------ ---- ---- -------

View File

@ -19,6 +19,8 @@ spec:
plural: canaries plural: canaries
singular: canary singular: canary
kind: Canary kind: Canary
categories:
- all
scope: Namespaced scope: Namespaced
subresources: subresources:
status: {} status: {}
@ -26,6 +28,9 @@ spec:
- name: Status - name: Status
type: string type: string
JSONPath: .status.phase JSONPath: .status.phase
- name: Weight
type: string
JSONPath: .status.canaryWeight
- name: LastTransitionTime - name: LastTransitionTime
type: string type: string
JSONPath: .status.lastTransitionTime JSONPath: .status.lastTransitionTime

View File

@ -20,6 +20,8 @@ spec:
plural: canaries plural: canaries
singular: canary singular: canary
kind: Canary kind: Canary
categories:
- all
scope: Namespaced scope: Namespaced
subresources: subresources:
status: {} status: {}
@ -27,6 +29,9 @@ spec:
- name: Status - name: Status
type: string type: string
JSONPath: .status.phase JSONPath: .status.phase
- name: Weight
type: string
JSONPath: .status.canaryWeight
- name: LastTransitionTime - name: LastTransitionTime
type: string type: string
JSONPath: .status.lastTransitionTime JSONPath: .status.lastTransitionTime

View File

@ -108,9 +108,9 @@ Flagger detects that the deployment revision changed and starts a new rollout:
kubectl -n test describe canary/podinfo kubectl -n test describe canary/podinfo
Status: Status:
Canary Revision: 19871136 Canary Weight: 0
Failed Checks: 0 Failed Checks: 0
State: finished Phase: Succeeded
Events: Events:
Type Reason Age From Message Type Reason Age From Message
---- ------ ---- ---- ------- ---- ------ ---- ---- -------
@ -132,6 +132,17 @@ Events:
Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test
``` ```
You can monitor all canaries with:
```bash
watch kubectl get canaries --all-namespaces
NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME
test podinfo Progressing 15 2019-01-16T14:05:07Z
prod frontend Succeeded 0 2019-01-15T16:15:07Z
prod backend Failed 0 2019-01-14T17:05:07Z
```
During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout. During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout.
Create a tester pod and exec into it: Create a tester pod and exec into it:
@ -162,9 +173,9 @@ When the number of failed checks reaches the canary analysis threshold, the traf
kubectl -n test describe canary/podinfo kubectl -n test describe canary/podinfo
Status: Status:
Canary Revision: 16695041 Canary Weight: 0
Failed Checks: 10 Failed Checks: 10
State: failed Phase: Failed
Events: Events:
Type Reason Age From Message Type Reason Age From Message
---- ------ ---- ---- ------- ---- ------ ---- ---- -------
@ -181,5 +192,3 @@ Events:
Warning Synced 1m flagger Canary failed! Scaling down podinfo.test Warning Synced 1m flagger Canary failed! Scaling down podinfo.test
``` ```
####

View File

@ -91,6 +91,7 @@ const (
type CanaryStatus struct { type CanaryStatus struct {
Phase CanaryPhase `json:"phase"` Phase CanaryPhase `json:"phase"`
FailedChecks int `json:"failedChecks"` FailedChecks int `json:"failedChecks"`
CanaryWeight int `json:"canaryWeight"`
// +optional // +optional
LastAppliedSpec string `json:"lastAppliedSpec,omitempty"` LastAppliedSpec string `json:"lastAppliedSpec,omitempty"`
// +optional // +optional

View File

@ -164,8 +164,8 @@ func (c *CanaryDeployer) ShouldAdvance(cd *flaggerv1.Canary) (bool, error) {
return c.IsNewSpec(cd) return c.IsNewSpec(cd)
} }
// SetFailedChecks updates the canary failed checks counter // SetStatusFailedChecks updates the canary failed checks counter
func (c *CanaryDeployer) SetFailedChecks(cd *flaggerv1.Canary, val int) error { func (c *CanaryDeployer) SetStatusFailedChecks(cd *flaggerv1.Canary, val int) error {
cdCopy := cd.DeepCopy() cdCopy := cd.DeepCopy()
cdCopy.Status.FailedChecks = val cdCopy.Status.FailedChecks = val
cdCopy.Status.LastTransitionTime = metav1.Now() cdCopy.Status.LastTransitionTime = metav1.Now()
@ -177,10 +177,10 @@ func (c *CanaryDeployer) SetFailedChecks(cd *flaggerv1.Canary, val int) error {
return nil return nil
} }
// SetState updates the canary status state // SetStatusWeight updates the canary status weight value
func (c *CanaryDeployer) SetState(cd *flaggerv1.Canary, state flaggerv1.CanaryPhase) error { func (c *CanaryDeployer) SetStatusWeight(cd *flaggerv1.Canary, val int) error {
cdCopy := cd.DeepCopy() cdCopy := cd.DeepCopy()
cdCopy.Status.Phase = state cdCopy.Status.CanaryWeight = val
cdCopy.Status.LastTransitionTime = metav1.Now() cdCopy.Status.LastTransitionTime = metav1.Now()
cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy) cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
@ -190,6 +190,23 @@ func (c *CanaryDeployer) SetState(cd *flaggerv1.Canary, state flaggerv1.CanaryPh
return nil return nil
} }
// SetStatusPhase updates the canary status phase
func (c *CanaryDeployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error {
cdCopy := cd.DeepCopy()
cdCopy.Status.Phase = phase
cdCopy.Status.LastTransitionTime = metav1.Now()
if phase != flaggerv1.CanaryProgressing {
cdCopy.Status.CanaryWeight = 0
}
cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
if err != nil {
return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
}
return nil
}
// SyncStatus encodes the canary pod spec and updates the canary status // SyncStatus encodes the canary pod spec and updates the canary status
func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatus) error { func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatus) error {
dep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(cd.Spec.TargetRef.Name, metav1.GetOptions{}) dep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(cd.Spec.TargetRef.Name, metav1.GetOptions{})
@ -207,6 +224,7 @@ func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.Canar
cdCopy := cd.DeepCopy() cdCopy := cd.DeepCopy()
cdCopy.Status.Phase = status.Phase cdCopy.Status.Phase = status.Phase
cdCopy.Status.CanaryWeight = status.CanaryWeight
cdCopy.Status.FailedChecks = status.FailedChecks cdCopy.Status.FailedChecks = status.FailedChecks
cdCopy.Status.LastAppliedSpec = base64.StdEncoding.EncodeToString(specJson) cdCopy.Status.LastAppliedSpec = base64.StdEncoding.EncodeToString(specJson)
cdCopy.Status.LastTransitionTime = metav1.Now() cdCopy.Status.LastTransitionTime = metav1.Now()

View File

@ -351,7 +351,7 @@ func TestCanaryDeployer_SetFailedChecks(t *testing.T) {
t.Fatal(err.Error()) t.Fatal(err.Error())
} }
err = deployer.SetFailedChecks(canary, 1) err = deployer.SetStatusFailedChecks(canary, 1)
if err != nil { if err != nil {
t.Fatal(err.Error()) t.Fatal(err.Error())
} }
@ -387,7 +387,7 @@ func TestCanaryDeployer_SetState(t *testing.T) {
t.Fatal(err.Error()) t.Fatal(err.Error())
} }
err = deployer.SetState(canary, v1alpha3.CanaryProgressing) err = deployer.SetStatusPhase(canary, v1alpha3.CanaryProgressing)
if err != nil { if err != nil {
t.Fatal(err.Error()) t.Fatal(err.Error())
} }

View File

@ -173,7 +173,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
} }
// mark canary as failed // mark canary as failed
if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed}); err != nil { if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed, CanaryWeight: 0}); err != nil {
c.logger.Errorf("%v", err) c.logger.Errorf("%v", err)
return return
} }
@ -188,7 +188,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
c.recordEventInfof(cd, "Starting canary deployment for %s.%s", cd.Name, cd.Namespace) c.recordEventInfof(cd, "Starting canary deployment for %s.%s", cd.Name, cd.Namespace)
} else { } else {
if ok := c.analyseCanary(cd); !ok { if ok := c.analyseCanary(cd); !ok {
if err := c.deployer.SetFailedChecks(cd, cd.Status.FailedChecks+1); err != nil { if err := c.deployer.SetStatusFailedChecks(cd, cd.Status.FailedChecks+1); err != nil {
c.recordEventWarningf(cd, "%v", err) c.recordEventWarningf(cd, "%v", err)
return return
} }
@ -212,6 +212,12 @@ func (c *Controller) advanceCanary(name string, namespace string) {
return return
} }
// update weight status
if err := c.deployer.SetStatusWeight(cd, canaryRoute.Weight); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
c.recorder.SetWeight(cd, primaryRoute.Weight, canaryRoute.Weight) c.recorder.SetWeight(cd, primaryRoute.Weight, canaryRoute.Weight)
c.recordEventInfof(cd, "Advance %s.%s canary weight %v", cd.Name, cd.Namespace, canaryRoute.Weight) c.recordEventInfof(cd, "Advance %s.%s canary weight %v", cd.Name, cd.Namespace, canaryRoute.Weight)
@ -243,8 +249,8 @@ func (c *Controller) advanceCanary(name string, namespace string) {
return return
} }
// update status // update status phase
if err := c.deployer.SetState(cd, flaggerv1.CanarySucceeded); err != nil { if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil {
c.recordEventWarningf(cd, "%v", err) c.recordEventWarningf(cd, "%v", err)
return return
} }