mirror of https://github.com/fluxcd/flagger.git
291 lines
9.2 KiB
Go
291 lines
9.2 KiB
Go
package canary
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
ex "github.com/pkg/errors"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/util/retry"
|
|
|
|
flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
|
|
clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
|
|
)
|
|
|
|
// SyncStatus encodes the canary pod spec and updates the canary status
|
|
func (c *DeploymentController) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatus) error {
|
|
dep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(cd.Spec.TargetRef.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
return fmt.Errorf("deployment %s.%s not found", cd.Spec.TargetRef.Name, cd.Namespace)
|
|
}
|
|
return ex.Wrap(err, "SyncStatus deployment query error")
|
|
}
|
|
|
|
configs, err := c.configTracker.GetConfigRefs(cd)
|
|
if err != nil {
|
|
return ex.Wrap(err, "SyncStatus configs query error")
|
|
}
|
|
|
|
return syncCanaryStatus(c.flaggerClient, cd, status, dep.Spec.Template, func(cdCopy *flaggerv1.Canary) {
|
|
cdCopy.Status.TrackedConfigs = configs
|
|
})
|
|
}
|
|
|
|
func syncCanaryStatus(flaggerClient clientset.Interface, cd *flaggerv1.Canary, status flaggerv1.CanaryStatus, canaryResource interface{}, setAll func(cdCopy *flaggerv1.Canary)) error {
|
|
hash := computeHash(canaryResource)
|
|
|
|
firstTry := true
|
|
err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
|
|
var selErr error
|
|
if !firstTry {
|
|
cd, selErr = flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
|
|
if selErr != nil {
|
|
return selErr
|
|
}
|
|
}
|
|
|
|
cdCopy := cd.DeepCopy()
|
|
cdCopy.Status.Phase = status.Phase
|
|
cdCopy.Status.CanaryWeight = status.CanaryWeight
|
|
cdCopy.Status.FailedChecks = status.FailedChecks
|
|
cdCopy.Status.Iterations = status.Iterations
|
|
cdCopy.Status.LastAppliedSpec = hash
|
|
cdCopy.Status.LastTransitionTime = metav1.Now()
|
|
setAll(cdCopy)
|
|
|
|
if ok, conditions := MakeStatusConditions(cd.Status, status.Phase); ok {
|
|
cdCopy.Status.Conditions = conditions
|
|
}
|
|
|
|
err = updateStatusWithUpgrade(flaggerClient, cdCopy)
|
|
firstTry = false
|
|
return
|
|
})
|
|
if err != nil {
|
|
return ex.Wrap(err, "SyncStatus")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SetStatusFailedChecks updates the canary failed checks counter
|
|
func (c *DeploymentController) SetStatusFailedChecks(cd *flaggerv1.Canary, val int) error {
|
|
return setStatusFailedChecks(c.flaggerClient, cd, val)
|
|
}
|
|
|
|
func setStatusFailedChecks(flaggerClient clientset.Interface, cd *flaggerv1.Canary, val int) error {
|
|
firstTry := true
|
|
err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
|
|
var selErr error
|
|
if !firstTry {
|
|
cd, selErr = flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
|
|
if selErr != nil {
|
|
return selErr
|
|
}
|
|
}
|
|
cdCopy := cd.DeepCopy()
|
|
cdCopy.Status.FailedChecks = val
|
|
cdCopy.Status.LastTransitionTime = metav1.Now()
|
|
|
|
err = updateStatusWithUpgrade(flaggerClient, cdCopy)
|
|
firstTry = false
|
|
return
|
|
})
|
|
if err != nil {
|
|
return ex.Wrap(err, "SetStatusFailedChecks")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SetStatusWeight updates the canary status weight value
|
|
func (c *DeploymentController) SetStatusWeight(cd *flaggerv1.Canary, val int) error {
|
|
return setStatusWeight(c.flaggerClient, cd, val)
|
|
}
|
|
|
|
func setStatusWeight(flaggerClient clientset.Interface, cd *flaggerv1.Canary, val int) error {
|
|
firstTry := true
|
|
err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
|
|
var selErr error
|
|
if !firstTry {
|
|
cd, selErr = flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
|
|
if selErr != nil {
|
|
return selErr
|
|
}
|
|
}
|
|
cdCopy := cd.DeepCopy()
|
|
cdCopy.Status.CanaryWeight = val
|
|
cdCopy.Status.LastTransitionTime = metav1.Now()
|
|
|
|
err = updateStatusWithUpgrade(flaggerClient, cdCopy)
|
|
firstTry = false
|
|
return
|
|
})
|
|
if err != nil {
|
|
return ex.Wrap(err, "SetStatusWeight")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SetStatusIterations updates the canary status iterations value
|
|
func (c *DeploymentController) SetStatusIterations(cd *flaggerv1.Canary, val int) error {
|
|
return setStatusIterations(c.flaggerClient, cd, val)
|
|
}
|
|
|
|
func setStatusIterations(flaggerClient clientset.Interface, cd *flaggerv1.Canary, val int) error {
|
|
firstTry := true
|
|
err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
|
|
var selErr error
|
|
if !firstTry {
|
|
cd, selErr = flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
|
|
if selErr != nil {
|
|
return selErr
|
|
}
|
|
}
|
|
|
|
cdCopy := cd.DeepCopy()
|
|
cdCopy.Status.Iterations = val
|
|
cdCopy.Status.LastTransitionTime = metav1.Now()
|
|
|
|
err = updateStatusWithUpgrade(flaggerClient, cdCopy)
|
|
firstTry = false
|
|
return
|
|
})
|
|
|
|
if err != nil {
|
|
return ex.Wrap(err, "SetStatusIterations")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SetStatusPhase updates the canary status phase
|
|
func (c *DeploymentController) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error {
|
|
return setStatusPhase(c.flaggerClient, cd, phase)
|
|
}
|
|
|
|
func setStatusPhase(flaggerClient clientset.Interface, cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error {
|
|
firstTry := true
|
|
err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
|
|
var selErr error
|
|
if !firstTry {
|
|
cd, selErr = flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
|
|
if selErr != nil {
|
|
return selErr
|
|
}
|
|
}
|
|
|
|
cdCopy := cd.DeepCopy()
|
|
cdCopy.Status.Phase = phase
|
|
cdCopy.Status.LastTransitionTime = metav1.Now()
|
|
|
|
if phase != flaggerv1.CanaryPhaseProgressing && phase != flaggerv1.CanaryPhaseWaiting {
|
|
cdCopy.Status.CanaryWeight = 0
|
|
cdCopy.Status.Iterations = 0
|
|
}
|
|
|
|
// on promotion set primary spec hash
|
|
if phase == flaggerv1.CanaryPhaseInitialized || phase == flaggerv1.CanaryPhaseSucceeded {
|
|
cdCopy.Status.LastPromotedSpec = cd.Status.LastAppliedSpec
|
|
}
|
|
|
|
if ok, conditions := MakeStatusConditions(cdCopy.Status, phase); ok {
|
|
cdCopy.Status.Conditions = conditions
|
|
}
|
|
|
|
err = updateStatusWithUpgrade(flaggerClient, cdCopy)
|
|
firstTry = false
|
|
return
|
|
})
|
|
if err != nil {
|
|
return ex.Wrap(err, "SetStatusPhase")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// getStatusCondition returns a condition based on type
|
|
func getStatusCondition(status flaggerv1.CanaryStatus, conditionType flaggerv1.CanaryConditionType) *flaggerv1.CanaryCondition {
|
|
for i := range status.Conditions {
|
|
c := status.Conditions[i]
|
|
if c.Type == conditionType {
|
|
return &c
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// MakeStatusCondition updates the canary status conditions based on canary phase
|
|
func MakeStatusConditions(canaryStatus flaggerv1.CanaryStatus,
|
|
phase flaggerv1.CanaryPhase) (bool, []flaggerv1.CanaryCondition) {
|
|
currentCondition := getStatusCondition(canaryStatus, flaggerv1.PromotedType)
|
|
|
|
message := "New deployment detected, starting initialization."
|
|
status := corev1.ConditionUnknown
|
|
switch phase {
|
|
case flaggerv1.CanaryPhaseInitializing:
|
|
status = corev1.ConditionUnknown
|
|
message = "New deployment detected, starting initialization."
|
|
case flaggerv1.CanaryPhaseInitialized:
|
|
status = corev1.ConditionTrue
|
|
message = "Deployment initialization completed."
|
|
case flaggerv1.CanaryPhaseWaiting:
|
|
status = corev1.ConditionUnknown
|
|
message = "Waiting for approval."
|
|
case flaggerv1.CanaryPhaseProgressing:
|
|
status = corev1.ConditionUnknown
|
|
message = "New revision detected, starting canary analysis."
|
|
case flaggerv1.CanaryPhasePromoting:
|
|
status = corev1.ConditionUnknown
|
|
message = "Canary analysis completed, starting primary rolling update."
|
|
case flaggerv1.CanaryPhaseFinalising:
|
|
status = corev1.ConditionUnknown
|
|
message = "Canary analysis completed, routing all traffic to primary."
|
|
case flaggerv1.CanaryPhaseSucceeded:
|
|
status = corev1.ConditionTrue
|
|
message = "Canary analysis completed successfully, promotion finished."
|
|
case flaggerv1.CanaryPhaseFailed:
|
|
status = corev1.ConditionFalse
|
|
message = "Canary analysis failed, deployment scaled to zero."
|
|
}
|
|
|
|
newCondition := &flaggerv1.CanaryCondition{
|
|
Type: flaggerv1.PromotedType,
|
|
Status: status,
|
|
LastUpdateTime: metav1.Now(),
|
|
LastTransitionTime: metav1.Now(),
|
|
Message: message,
|
|
Reason: string(phase),
|
|
}
|
|
|
|
if currentCondition != nil &&
|
|
currentCondition.Status == newCondition.Status &&
|
|
currentCondition.Reason == newCondition.Reason {
|
|
return false, nil
|
|
}
|
|
|
|
if currentCondition != nil && currentCondition.Status == newCondition.Status {
|
|
newCondition.LastTransitionTime = currentCondition.LastTransitionTime
|
|
}
|
|
|
|
return true, []flaggerv1.CanaryCondition{*newCondition}
|
|
}
|
|
|
|
// updateStatusWithUpgrade tries to update the status sub-resource
|
|
// if the status update fails with:
|
|
// Canary.flagger.app is invalid: apiVersion: Invalid value: flagger.app/v1alpha3: must be flagger.app/v1beta1
|
|
// then the canary object will be updated to the latest API version
|
|
func updateStatusWithUpgrade(flaggerClient clientset.Interface, cd *flaggerv1.Canary) error {
|
|
_, err := flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).UpdateStatus(cd)
|
|
if err != nil && strings.Contains(err.Error(), "flagger.app/v1alpha") {
|
|
// upgrade alpha resource
|
|
_, updateErr := flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).Update(cd)
|
|
if updateErr != nil {
|
|
return updateErr
|
|
}
|
|
|
|
// retry status update
|
|
_, err = flaggerClient.FlaggerV1beta1().Canaries(cd.Namespace).UpdateStatus(cd)
|
|
}
|
|
return err
|
|
}
|