mirror of https://github.com/fluxcd/flagger.git
Add traffic weight to canary status
- show current weight on kubectl get canaries and kubectl get all
This commit is contained in:
parent
acdd2c46d5
commit
1ef310f00d
25
README.md
25
README.md
|
@ -242,15 +242,16 @@ kubectl -n test set image deployment/podinfo \
|
||||||
podinfod=quay.io/stefanprodan/podinfo:1.2.1
|
podinfod=quay.io/stefanprodan/podinfo:1.2.1
|
||||||
```
|
```
|
||||||
|
|
||||||
Flagger detects that the deployment revision changed and starts a new rollout:
|
Flagger detects that the deployment revision changed and starts a new canary analysis:
|
||||||
|
|
||||||
```
|
```
|
||||||
kubectl -n test describe canary/podinfo
|
kubectl -n test describe canary/podinfo
|
||||||
|
|
||||||
Status:
|
Status:
|
||||||
Canary Revision: 19871136
|
Canary Weight: 0
|
||||||
Failed Checks: 0
|
Failed Checks: 0
|
||||||
State: finished
|
Last Transition Time: 2019-01-16T13:47:16Z
|
||||||
|
Phase: Succeeded
|
||||||
Events:
|
Events:
|
||||||
Type Reason Age From Message
|
Type Reason Age From Message
|
||||||
---- ------ ---- ---- -------
|
---- ------ ---- ---- -------
|
||||||
|
@ -272,6 +273,15 @@ Events:
|
||||||
Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test
|
Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test
|
||||||
```
|
```
|
||||||
|
|
||||||
|
You can monitor all canaries with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
watch kubectl get canaries --all-namespaces
|
||||||
|
|
||||||
|
NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME
|
||||||
|
test podinfo Progressing 5 2019-01-16T14:05:07Z
|
||||||
|
```
|
||||||
|
|
||||||
During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout.
|
During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout.
|
||||||
|
|
||||||
Create a tester pod and exec into it:
|
Create a tester pod and exec into it:
|
||||||
|
@ -300,9 +310,10 @@ the canary is scaled to zero and the rollout is marked as failed.
|
||||||
kubectl -n test describe canary/podinfo
|
kubectl -n test describe canary/podinfo
|
||||||
|
|
||||||
Status:
|
Status:
|
||||||
Canary Revision: 16695041
|
Canary Weight: 0
|
||||||
Failed Checks: 10
|
Failed Checks: 10
|
||||||
State: failed
|
Last Transition Time: 2019-01-16T13:47:16Z
|
||||||
|
Phase: Failed
|
||||||
Events:
|
Events:
|
||||||
Type Reason Age From Message
|
Type Reason Age From Message
|
||||||
---- ------ ---- ---- -------
|
---- ------ ---- ---- -------
|
||||||
|
|
|
@ -19,6 +19,8 @@ spec:
|
||||||
plural: canaries
|
plural: canaries
|
||||||
singular: canary
|
singular: canary
|
||||||
kind: Canary
|
kind: Canary
|
||||||
|
categories:
|
||||||
|
- all
|
||||||
scope: Namespaced
|
scope: Namespaced
|
||||||
subresources:
|
subresources:
|
||||||
status: {}
|
status: {}
|
||||||
|
@ -26,6 +28,9 @@ spec:
|
||||||
- name: Status
|
- name: Status
|
||||||
type: string
|
type: string
|
||||||
JSONPath: .status.phase
|
JSONPath: .status.phase
|
||||||
|
- name: Weight
|
||||||
|
type: string
|
||||||
|
JSONPath: .status.canaryWeight
|
||||||
- name: LastTransitionTime
|
- name: LastTransitionTime
|
||||||
type: string
|
type: string
|
||||||
JSONPath: .status.lastTransitionTime
|
JSONPath: .status.lastTransitionTime
|
||||||
|
|
|
@ -20,6 +20,8 @@ spec:
|
||||||
plural: canaries
|
plural: canaries
|
||||||
singular: canary
|
singular: canary
|
||||||
kind: Canary
|
kind: Canary
|
||||||
|
categories:
|
||||||
|
- all
|
||||||
scope: Namespaced
|
scope: Namespaced
|
||||||
subresources:
|
subresources:
|
||||||
status: {}
|
status: {}
|
||||||
|
@ -27,6 +29,9 @@ spec:
|
||||||
- name: Status
|
- name: Status
|
||||||
type: string
|
type: string
|
||||||
JSONPath: .status.phase
|
JSONPath: .status.phase
|
||||||
|
- name: Weight
|
||||||
|
type: string
|
||||||
|
JSONPath: .status.canaryWeight
|
||||||
- name: LastTransitionTime
|
- name: LastTransitionTime
|
||||||
type: string
|
type: string
|
||||||
JSONPath: .status.lastTransitionTime
|
JSONPath: .status.lastTransitionTime
|
||||||
|
|
|
@ -108,9 +108,9 @@ Flagger detects that the deployment revision changed and starts a new rollout:
|
||||||
kubectl -n test describe canary/podinfo
|
kubectl -n test describe canary/podinfo
|
||||||
|
|
||||||
Status:
|
Status:
|
||||||
Canary Revision: 19871136
|
Canary Weight: 0
|
||||||
Failed Checks: 0
|
Failed Checks: 0
|
||||||
State: finished
|
Phase: Succeeded
|
||||||
Events:
|
Events:
|
||||||
Type Reason Age From Message
|
Type Reason Age From Message
|
||||||
---- ------ ---- ---- -------
|
---- ------ ---- ---- -------
|
||||||
|
@ -132,6 +132,17 @@ Events:
|
||||||
Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test
|
Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test
|
||||||
```
|
```
|
||||||
|
|
||||||
|
You can monitor all canaries with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
watch kubectl get canaries --all-namespaces
|
||||||
|
|
||||||
|
NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME
|
||||||
|
test podinfo Progressing 15 2019-01-16T14:05:07Z
|
||||||
|
prod frontend Succeeded 0 2019-01-15T16:15:07Z
|
||||||
|
prod backend Failed 0 2019-01-14T17:05:07Z
|
||||||
|
```
|
||||||
|
|
||||||
During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout.
|
During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout.
|
||||||
|
|
||||||
Create a tester pod and exec into it:
|
Create a tester pod and exec into it:
|
||||||
|
@ -162,9 +173,9 @@ When the number of failed checks reaches the canary analysis threshold, the traf
|
||||||
kubectl -n test describe canary/podinfo
|
kubectl -n test describe canary/podinfo
|
||||||
|
|
||||||
Status:
|
Status:
|
||||||
Canary Revision: 16695041
|
Canary Weight: 0
|
||||||
Failed Checks: 10
|
Failed Checks: 10
|
||||||
State: failed
|
Phase: Failed
|
||||||
Events:
|
Events:
|
||||||
Type Reason Age From Message
|
Type Reason Age From Message
|
||||||
---- ------ ---- ---- -------
|
---- ------ ---- ---- -------
|
||||||
|
@ -181,5 +192,3 @@ Events:
|
||||||
Warning Synced 1m flagger Canary failed! Scaling down podinfo.test
|
Warning Synced 1m flagger Canary failed! Scaling down podinfo.test
|
||||||
```
|
```
|
||||||
|
|
||||||
####
|
|
||||||
|
|
||||||
|
|
|
@ -91,6 +91,7 @@ const (
|
||||||
type CanaryStatus struct {
|
type CanaryStatus struct {
|
||||||
Phase CanaryPhase `json:"phase"`
|
Phase CanaryPhase `json:"phase"`
|
||||||
FailedChecks int `json:"failedChecks"`
|
FailedChecks int `json:"failedChecks"`
|
||||||
|
CanaryWeight int `json:"canaryWeight"`
|
||||||
// +optional
|
// +optional
|
||||||
LastAppliedSpec string `json:"lastAppliedSpec,omitempty"`
|
LastAppliedSpec string `json:"lastAppliedSpec,omitempty"`
|
||||||
// +optional
|
// +optional
|
||||||
|
|
|
@ -164,8 +164,8 @@ func (c *CanaryDeployer) ShouldAdvance(cd *flaggerv1.Canary) (bool, error) {
|
||||||
return c.IsNewSpec(cd)
|
return c.IsNewSpec(cd)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetFailedChecks updates the canary failed checks counter
|
// SetStatusFailedChecks updates the canary failed checks counter
|
||||||
func (c *CanaryDeployer) SetFailedChecks(cd *flaggerv1.Canary, val int) error {
|
func (c *CanaryDeployer) SetStatusFailedChecks(cd *flaggerv1.Canary, val int) error {
|
||||||
cdCopy := cd.DeepCopy()
|
cdCopy := cd.DeepCopy()
|
||||||
cdCopy.Status.FailedChecks = val
|
cdCopy.Status.FailedChecks = val
|
||||||
cdCopy.Status.LastTransitionTime = metav1.Now()
|
cdCopy.Status.LastTransitionTime = metav1.Now()
|
||||||
|
@ -177,10 +177,10 @@ func (c *CanaryDeployer) SetFailedChecks(cd *flaggerv1.Canary, val int) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetState updates the canary status state
|
// SetStatusWeight updates the canary status weight value
|
||||||
func (c *CanaryDeployer) SetState(cd *flaggerv1.Canary, state flaggerv1.CanaryPhase) error {
|
func (c *CanaryDeployer) SetStatusWeight(cd *flaggerv1.Canary, val int) error {
|
||||||
cdCopy := cd.DeepCopy()
|
cdCopy := cd.DeepCopy()
|
||||||
cdCopy.Status.Phase = state
|
cdCopy.Status.CanaryWeight = val
|
||||||
cdCopy.Status.LastTransitionTime = metav1.Now()
|
cdCopy.Status.LastTransitionTime = metav1.Now()
|
||||||
|
|
||||||
cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
|
cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
|
||||||
|
@ -190,6 +190,23 @@ func (c *CanaryDeployer) SetState(cd *flaggerv1.Canary, state flaggerv1.CanaryPh
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetStatusPhase updates the canary status phase
|
||||||
|
func (c *CanaryDeployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error {
|
||||||
|
cdCopy := cd.DeepCopy()
|
||||||
|
cdCopy.Status.Phase = phase
|
||||||
|
cdCopy.Status.LastTransitionTime = metav1.Now()
|
||||||
|
|
||||||
|
if phase != flaggerv1.CanaryProgressing {
|
||||||
|
cdCopy.Status.CanaryWeight = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// SyncStatus encodes the canary pod spec and updates the canary status
|
// SyncStatus encodes the canary pod spec and updates the canary status
|
||||||
func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatus) error {
|
func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatus) error {
|
||||||
dep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(cd.Spec.TargetRef.Name, metav1.GetOptions{})
|
dep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(cd.Spec.TargetRef.Name, metav1.GetOptions{})
|
||||||
|
@ -207,6 +224,7 @@ func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.Canar
|
||||||
|
|
||||||
cdCopy := cd.DeepCopy()
|
cdCopy := cd.DeepCopy()
|
||||||
cdCopy.Status.Phase = status.Phase
|
cdCopy.Status.Phase = status.Phase
|
||||||
|
cdCopy.Status.CanaryWeight = status.CanaryWeight
|
||||||
cdCopy.Status.FailedChecks = status.FailedChecks
|
cdCopy.Status.FailedChecks = status.FailedChecks
|
||||||
cdCopy.Status.LastAppliedSpec = base64.StdEncoding.EncodeToString(specJson)
|
cdCopy.Status.LastAppliedSpec = base64.StdEncoding.EncodeToString(specJson)
|
||||||
cdCopy.Status.LastTransitionTime = metav1.Now()
|
cdCopy.Status.LastTransitionTime = metav1.Now()
|
||||||
|
|
|
@ -351,7 +351,7 @@ func TestCanaryDeployer_SetFailedChecks(t *testing.T) {
|
||||||
t.Fatal(err.Error())
|
t.Fatal(err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
err = deployer.SetFailedChecks(canary, 1)
|
err = deployer.SetStatusFailedChecks(canary, 1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err.Error())
|
t.Fatal(err.Error())
|
||||||
}
|
}
|
||||||
|
@ -387,7 +387,7 @@ func TestCanaryDeployer_SetState(t *testing.T) {
|
||||||
t.Fatal(err.Error())
|
t.Fatal(err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
err = deployer.SetState(canary, v1alpha3.CanaryProgressing)
|
err = deployer.SetStatusPhase(canary, v1alpha3.CanaryProgressing)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err.Error())
|
t.Fatal(err.Error())
|
||||||
}
|
}
|
||||||
|
|
|
@ -173,7 +173,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// mark canary as failed
|
// mark canary as failed
|
||||||
if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed}); err != nil {
|
if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed, CanaryWeight: 0}); err != nil {
|
||||||
c.logger.Errorf("%v", err)
|
c.logger.Errorf("%v", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -188,7 +188,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
|
||||||
c.recordEventInfof(cd, "Starting canary deployment for %s.%s", cd.Name, cd.Namespace)
|
c.recordEventInfof(cd, "Starting canary deployment for %s.%s", cd.Name, cd.Namespace)
|
||||||
} else {
|
} else {
|
||||||
if ok := c.analyseCanary(cd); !ok {
|
if ok := c.analyseCanary(cd); !ok {
|
||||||
if err := c.deployer.SetFailedChecks(cd, cd.Status.FailedChecks+1); err != nil {
|
if err := c.deployer.SetStatusFailedChecks(cd, cd.Status.FailedChecks+1); err != nil {
|
||||||
c.recordEventWarningf(cd, "%v", err)
|
c.recordEventWarningf(cd, "%v", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -212,6 +212,12 @@ func (c *Controller) advanceCanary(name string, namespace string) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// update weight status
|
||||||
|
if err := c.deployer.SetStatusWeight(cd, canaryRoute.Weight); err != nil {
|
||||||
|
c.recordEventWarningf(cd, "%v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
c.recorder.SetWeight(cd, primaryRoute.Weight, canaryRoute.Weight)
|
c.recorder.SetWeight(cd, primaryRoute.Weight, canaryRoute.Weight)
|
||||||
c.recordEventInfof(cd, "Advance %s.%s canary weight %v", cd.Name, cd.Namespace, canaryRoute.Weight)
|
c.recordEventInfof(cd, "Advance %s.%s canary weight %v", cd.Name, cd.Namespace, canaryRoute.Weight)
|
||||||
|
|
||||||
|
@ -243,8 +249,8 @@ func (c *Controller) advanceCanary(name string, namespace string) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// update status
|
// update status phase
|
||||||
if err := c.deployer.SetState(cd, flaggerv1.CanarySucceeded); err != nil {
|
if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil {
|
||||||
c.recordEventWarningf(cd, "%v", err)
|
c.recordEventWarningf(cd, "%v", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue