support bluegreen release: webhook update

Signed-off-by: yunbo <yunbo10124scut@gmail.com>

pkg/controller/batchrelease/control/bluegreenstyle/cloneset/control.go

@@ -154,10 +154,6 @@ func (rc *realController) Finalize(release *v1beta1.BatchRelease) error {
 		return errors.NewFatalError(fmt.Errorf("cannot get original setting for cloneset %v: %s from annotation", klog.KObj(rc.object), err.Error()))
 	}
 	patchData := patch.NewClonesetPatch()
-	// why we need a simple MinReadySeconds-based status machine? (ie. the if-else block)
-	// It's possible for Finalize to be called multiple times, if error returned is not nil.
-	// if we do all needed operations in a single code block, like, A->B->C, when C need retry,
-	// both A and B will be executed as well, however, operations like restoreHPA cost a lot(which calls LIST API)
 	if rc.object.Spec.MinReadySeconds != setting.MinReadySeconds {
 		// restore the hpa
 		if err := hpa.RestoreHPA(rc.client, rc.object); err != nil {
@@ -170,21 +166,18 @@ func (rc *realController) Finalize(release *v1beta1.BatchRelease) error {
 		if err := rc.client.Patch(context.TODO(), c, patchData); err != nil {
 			return err
 		}
-		// we should return an error to trigger re-enqueue, so that we can go to the next if-else branch in the next reconcile
-		return errors.NewBenignError(fmt.Errorf("cloneset bluegreen: we should wait all pods updated and available"))
-	} else {
-		klog.InfoS("Finalize: cloneset bluegreen release: wait all pods updated and ready", "cloneset", klog.KObj(rc.object))
-		// wait all pods updated and ready
-		if rc.object.Status.ReadyReplicas != rc.object.Status.UpdatedReadyReplicas {
-			return errors.NewBenignError(fmt.Errorf("cloneset %v finalize not done, readyReplicas %d != updatedReadyReplicas %d, current policy %s",
-				klog.KObj(rc.object), rc.object.Status.ReadyReplicas, rc.object.Status.UpdatedReadyReplicas, release.Spec.ReleasePlan.FinalizingPolicy))
-		}
-		klog.InfoS("Finalize: cloneset bluegreen release: all pods updated and ready")
-		// restore annotation
-		patchData.DeleteAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation)
-		patchData.DeleteAnnotation(util.BatchReleaseControlAnnotation)
-		return rc.client.Patch(context.TODO(), c, patchData)
 	}
+	klog.InfoS("Finalize: cloneset bluegreen release: wait all pods updated and ready", "cloneset", klog.KObj(rc.object))
+	// wait all pods updated and ready
+	if rc.object.Status.ReadyReplicas != rc.object.Status.UpdatedReadyReplicas {
+		return errors.NewBenignError(fmt.Errorf("cloneset %v finalize not done, readyReplicas %d != updatedReadyReplicas %d, current policy %s",
+			klog.KObj(rc.object), rc.object.Status.ReadyReplicas, rc.object.Status.UpdatedReadyReplicas, release.Spec.ReleasePlan.FinalizingPolicy))
+	}
+	klog.InfoS("Finalize: cloneset bluegreen release: all pods updated and ready")
+	// restore annotation
+	patchData.DeleteAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation)
+	patchData.DeleteAnnotation(util.BatchReleaseControlAnnotation)
+	return rc.client.Patch(context.TODO(), c, patchData)
 }
 
 func (rc *realController) finalized() bool {

pkg/controller/batchrelease/control/bluegreenstyle/deployment/control.go

@@ -47,12 +47,14 @@ type realController struct {
 	pods   []*corev1.Pod
 	key    types.NamespacedName
 	object *apps.Deployment
+	finder *util.ControllerFinder
 }
 
 func NewController(cli client.Client, key types.NamespacedName, _ schema.GroupVersionKind) bluegreenstyle.Interface {
 	return &realController{
 		key:    key,
 		client: cli,
+		finder: util.NewControllerFinder(cli),
 	}
 }
 
@@ -82,39 +84,29 @@ func (rc *realController) ListOwnedPods() ([]*corev1.Pod, error) {
 	return rc.pods, err
 }
 
-// Add OriginalDeploymentStrategyAnnotation to workload
+// Initialize prepares the Deployment for the BatchRelease process
 func (rc *realController) Initialize(release *v1beta1.BatchRelease) error {
 	if rc.object == nil || control.IsControlledByBatchRelease(release, rc.object) {
 		return nil
 	}
-	// disable the hpa
+	// Disable the HPA
 	if err := hpa.DisableHPA(rc.client, rc.object); err != nil {
 		return err
 	}
-	klog.InfoS("Initialize: disable hpa for deployment successfully", "deployment", klog.KObj(rc.object))
+	klog.InfoS("Initialize: disabled HPA for deployment successfully", "deployment", klog.KObj(rc.object))
-	// update the deployment
-	setting, err := control.GetOriginalSetting(rc.object)
-	if err != nil {
-		return errors.NewFatalError(fmt.Errorf("cannot get original setting for cloneset %v: %s from annotation", klog.KObj(rc.object), err.Error()))
-	}
-	control.InitOriginalSetting(&setting, rc.object)
-	klog.InfoS("Initialize deployment", "deployment", klog.KObj(rc.object), "setting", util.DumpJSON(&setting))
 
-	patchData := patch.NewDeploymentPatch()
+	// Patch minReadySeconds for stable ReplicaSet
-	patchData.InsertAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation, util.DumpJSON(&setting))
+	if err := rc.patchStableRSMinReadySeconds(v1beta1.MaxReadySeconds); err != nil {
-	patchData.InsertAnnotation(util.BatchReleaseControlAnnotation, util.DumpJSON(metav1.NewControllerRef(
+		return err
-		release, release.GetObjectKind().GroupVersionKind())))
+	}
-	// update: MinReadySeconds, ProgressDeadlineSeconds, MaxSurge, MaxUnavailable
+	klog.InfoS("Initialize: patched minReadySeconds for stable replicaset successfully", "deployment", klog.KObj(rc.object))
-	patchData.UpdateStrategy(apps.DeploymentStrategy{
+
-		Type: apps.RollingUpdateDeploymentStrategyType,
+	// Patch Deplopyment
-		RollingUpdate: &apps.RollingUpdateDeployment{
+	if err := rc.patchDeployment(release); err != nil {
-			MaxSurge:       &intstr.IntOrString{Type: intstr.Int, IntVal: 1},
+		return err
-			MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 0},
+	}
-		},
+	klog.InfoS("Initialize: patched deployment successfully", "deployment", klog.KObj(rc.object))
-	})
+	return nil
-	patchData.UpdateMinReadySeconds(v1beta1.MaxReadySeconds)
-	patchData.UpdateProgressDeadlineSeconds(utilpointer.Int32(v1beta1.MaxProgressSeconds))
-	return rc.client.Patch(context.TODO(), util.GetEmptyObjectWithKey(rc.object), patchData)
 }
 
 func (rc *realController) UpgradeBatch(ctx *batchcontext.BatchContext) error {
@@ -131,6 +123,7 @@ func (rc *realController) UpgradeBatch(ctx *batchcontext.BatchContext) error {
 	klog.Infof("Ready to upgrade batch for deployment %v: current %d < desired %d", klog.KObj(rc.object), current, desired)
 	patchData := patch.NewDeploymentPatch()
 	// different with canary release, bluegreen don't need to set pause in the process of rollout
+	// because our webhook may pause the Deployment in some situations, we ensure that the Deployment is not paused
 	patchData.UpdatePaused(false)
 	patchData.UpdateStrategy(apps.DeploymentStrategy{
 		Type: apps.RollingUpdateDeploymentStrategyType,
@@ -166,10 +159,6 @@ func (rc *realController) Finalize(release *v1beta1.BatchRelease) error {
 		return errors.NewFatalError(fmt.Errorf("cannot get original setting for cloneset %v: %s from annotation", klog.KObj(rc.object), err.Error()))
 	}
 	patchData := patch.NewDeploymentPatch()
-	// why we need a simple MinReadySeconds-based status machine? (ie. the if-else block)
-	// It's possible for Finalize to be called multiple times, if error returned is not nil.
-	// if we do all needed operations in a single code block, like, A->B->C, when C need retry,
-	// both A and B will be executed as well, however, operations like restoreHPA cost a lot(which calls LIST API)
 	if rc.object.Spec.MinReadySeconds != setting.MinReadySeconds {
 		// restore the hpa
 		if err := hpa.RestoreHPA(rc.client, rc.object); err != nil {
@@ -184,21 +173,18 @@ func (rc *realController) Finalize(release *v1beta1.BatchRelease) error {
 		if err := rc.client.Patch(context.TODO(), d, patchData); err != nil {
 			return err
 		}
-		// we should return an error to trigger re-enqueue, so that we can go to the next if-else branch in the next reconcile
-		return errors.NewBenignError(fmt.Errorf("deployment bluegreen: we should wait all pods updated and available"))
-	} else {
-		klog.InfoS("Finalize: deployment bluegreen release: wait all pods updated and ready", "cloneset", klog.KObj(rc.object))
-		// wait all pods updated and ready
-		if err := waitAllUpdatedAndReady(d.(*apps.Deployment)); err != nil {
-			return errors.NewBenignError(err)
-		}
-		klog.InfoS("Finalize: deployment is ready to resume, restore the original setting", "deployment", klog.KObj(rc.object))
-		// restore label and annotation
-		patchData.DeleteAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation)
-		patchData.DeleteLabel(v1alpha1.DeploymentStableRevisionLabel)
-		patchData.DeleteAnnotation(util.BatchReleaseControlAnnotation)
-		return rc.client.Patch(context.TODO(), d, patchData)
 	}
+	klog.InfoS("Finalize: deployment bluegreen release: wait all pods updated and ready", "cloneset", klog.KObj(rc.object))
+	// wait all pods updated and ready
+	if err := waitAllUpdatedAndReady(d.(*apps.Deployment)); err != nil {
+		return errors.NewBenignError(err)
+	}
+	klog.InfoS("Finalize: deployment is ready to resume, restore the original setting", "deployment", klog.KObj(rc.object))
+	// restore label and annotation
+	patchData.DeleteAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation)
+	patchData.DeleteLabel(v1alpha1.DeploymentStableRevisionLabel)
+	patchData.DeleteAnnotation(util.BatchReleaseControlAnnotation)
+	return rc.client.Patch(context.TODO(), d, patchData)
 }
 
 func (rc *realController) finalized() bool {
@@ -301,3 +287,53 @@ func waitAllUpdatedAndReady(deployment *apps.Deployment) error {
 	}
 	return nil
 }
+
+// Patch minReadySeconds for stable ReplicaSet
+/*
+	Here is why:
+	For rollback scenario, we should set the stable rs minReadySeconds to infinity to make pods of the stable rs unavailable,
+	otherwise Pods in new version would be terminated immediately when rollback happens.
+	we want to keep them until traffic is switched to the stable version
+*/
+func (rc *realController) patchStableRSMinReadySeconds(seconds int32) error {
+	if stableRS, err := rc.finder.GetDeploymentStableRs(rc.object); err != nil {
+		return fmt.Errorf("failed to get stable ReplicaSet: %v", err)
+	} else if stableRS == nil {
+		klog.Warningf("No stable ReplicaSet found for deployment %s/%s", rc.object.Namespace, rc.object.Name)
+	} else {
+		body := fmt.Sprintf(`{"spec":{"minReadySeconds":%v}}`, seconds)
+		if err = rc.client.Patch(context.TODO(), stableRS, client.RawPatch(types.MergePatchType, []byte(body))); err != nil {
+			return fmt.Errorf("failed to patch ReplicaSet %s/%s minReadySeconds to %v: %v", stableRS.Namespace, stableRS.Name, v1beta1.MaxReadySeconds, err)
+		}
+	}
+	return nil
+}
+
+// Update deployment strategy: MinReadySeconds, ProgressDeadlineSeconds, MaxSurge, MaxUnavailable
+func (rc *realController) patchDeployment(release *v1beta1.BatchRelease) error {
+	setting, err := control.GetOriginalSetting(rc.object)
+	if err != nil {
+		return errors.NewFatalError(fmt.Errorf("cannot get original setting for deployment %v: %s", klog.KObj(rc.object), err.Error()))
+	}
+	control.InitOriginalSetting(&setting, rc.object)
+	patchData := patch.NewDeploymentPatch()
+	patchData.InsertAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation, util.DumpJSON(&setting))
+	patchData.InsertAnnotation(util.BatchReleaseControlAnnotation, util.DumpJSON(metav1.NewControllerRef(
+		release, release.GetObjectKind().GroupVersionKind())))
+
+	patchData.UpdateStrategy(apps.DeploymentStrategy{
+		Type: apps.RollingUpdateDeploymentStrategyType,
+		RollingUpdate: &apps.RollingUpdateDeployment{
+			MaxSurge:       &intstr.IntOrString{Type: intstr.Int, IntVal: 1},
+			MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 0},
+		},
+	})
+	patchData.UpdateMinReadySeconds(v1beta1.MaxReadySeconds)
+	patchData.UpdateProgressDeadlineSeconds(utilpointer.Int32(v1beta1.MaxProgressSeconds))
+
+	// Apply the patch to the Deployment
+	if err := rc.client.Patch(context.TODO(), util.GetEmptyObjectWithKey(rc.object), patchData); err != nil {
+		return fmt.Errorf("failed to patch deployment %v: %v", klog.KObj(rc.object), err)
+	}
+	return nil
+}

pkg/controller/batchrelease/control/bluegreenstyle/deployment/control_test.go

@@ -23,6 +23,7 @@ import (
 	"reflect"
 	"strconv"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -388,7 +389,8 @@ func TestRealController(t *testing.T) {
 
 	release := releaseDemo.DeepCopy()
 	clone := deploymentDemo.DeepCopy()
-	cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(release, clone).Build()
+	stableRs, canaryRs := makeStableReplicaSets(clone), makeCanaryReplicaSets(clone)
+	cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(release, clone, stableRs, canaryRs).Build()
 	// build new controller
 	c := NewController(cli, deploymentKey, clone.GroupVersionKind()).(*realController)
 	controller, err := c.BuildController()
@@ -414,6 +416,10 @@ func TestRealController(t *testing.T) {
 		MinReadySeconds:         0,
 		ProgressDeadlineSeconds: pointer.Int32(600),
 	})))
+	// check minReadyseconds field of stable replicaset
+	fetchRS := &apps.ReplicaSet{}
+	Expect(cli.Get(context.TODO(), types.NamespacedName{Name: stableRs.GetName(), Namespace: stableRs.GetNamespace()}, fetchRS)).NotTo(HaveOccurred())
+	Expect(fetchRS.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds)))
 
 	c.object = fetch // mock
 
@@ -469,6 +475,7 @@ func makeCanaryReplicaSets(d client.Object) client.Object {
 			OwnerReferences: []metav1.OwnerReference{
 				*metav1.NewControllerRef(deploy, deploy.GroupVersionKind()),
 			},
+			CreationTimestamp: metav1.Now(),
 		},
 		Spec: apps.ReplicaSetSpec{
 			Replicas: deploy.Spec.Replicas,
@@ -498,6 +505,7 @@ func makeStableReplicaSets(d client.Object) client.Object {
 			OwnerReferences: []metav1.OwnerReference{
 				*metav1.NewControllerRef(deploy, deploy.GroupVersionKind()),
 			},
+			CreationTimestamp: metav1.NewTime(time.Now().Add(-time.Hour)),
 		},
 		Spec: apps.ReplicaSetSpec{
 			Replicas: deploy.Spec.Replicas,

pkg/controller/rollout/rollout_progressing.go

@@ -25,6 +25,7 @@ import (
 	"github.com/openkruise/rollouts/api/v1beta1"
 	"github.com/openkruise/rollouts/pkg/trafficrouting"
 	"github.com/openkruise/rollouts/pkg/util"
+	utilerrors "github.com/openkruise/rollouts/pkg/util/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -110,7 +111,12 @@ func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *v1beta1.Rollout
 	case v1alpha1.ProgressingReasonInRolling:
 		klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason)
 		err = r.doProgressingInRolling(rolloutContext)
-		if err != nil {
+		if utilerrors.IsFatal(err) {
+			// For fatal errors, do not retry as it wastes resources and has no effect.
+			// therefore, we don't propagate the error, but just log it.
+			// user should do sth instead, eg. for bluegreen continuous release scenario, user should do rollback
+			klog.Warningf("rollout(%s/%s) doProgressingInRolling error(%s)", rollout.Namespace, rollout.Name, err.Error())
+		} else if err != nil {
 			return nil, err
 		}
 
@@ -230,6 +236,14 @@ func (r *RolloutReconciler) handleContinuousRelease(c *RolloutContext) error {
 	klog.Infof("rollout(%s/%s) workload continuous publishing canaryRevision from(%s) -> to(%s), then restart publishing",
 		c.Rollout.Namespace, c.Rollout.Name, c.NewStatus.GetCanaryRevision(), c.Workload.CanaryRevision)
 
+	// do nothing for blue-green release
+	if c.Rollout.Spec.Strategy.IsBlueGreenRelease() {
+		cond := util.GetRolloutCondition(*c.NewStatus, v1beta1.RolloutConditionProgressing)
+		cond.Message = "[warning] new version released in progress of blue-green release, please rollback first"
+		c.NewStatus.Message = cond.Message
+		return utilerrors.NewFatalError(fmt.Errorf("cannot do continuous release for blue-green release, rollback firstly"))
+	}
+
 	done, err := r.doProgressingReset(c)
 	if err != nil {
 		klog.Errorf("rollout(%s/%s) doProgressingReset failed: %s", c.Rollout.Namespace, c.Rollout.Name, err.Error())

pkg/util/client/delegating_client.go

@@ -153,12 +153,7 @@ func (d *delegatingReader) List(ctx context.Context, list client.ObjectList, opt
 	return d.CacheReader.List(ctx, list, opts...)
 }
 
-var DisableDeepCopy = disableDeepCopy{}
+var DisableDeepCopy = client.UnsafeDisableDeepCopy
-
-type disableDeepCopy struct{}
-
-func (_ disableDeepCopy) ApplyToList(_ *client.ListOptions) {
-}
 
 func isDisableDeepCopy(opts []client.ListOption) bool {
 	for _, opt := range opts {

pkg/util/controller_finder.go

@@ -289,7 +289,7 @@ func (r *ControllerFinder) getDeployment(namespace string, ref *rolloutv1beta1.O
 		return &Workload{IsStatusConsistent: false}, nil
 	}
 	// stable replicaSet
-	stableRs, err := r.getDeploymentStableRs(stable)
+	stableRs, err := r.GetDeploymentStableRs(stable)
 	if err != nil || stableRs == nil {
 		return &Workload{IsStatusConsistent: false}, err
 	}
@@ -318,7 +318,7 @@ func (r *ControllerFinder) getDeployment(namespace string, ref *rolloutv1beta1.O
 	if err != nil || canary == nil {
 		return workload, err
 	}
-	canaryRs, err := r.getDeploymentStableRs(canary)
+	canaryRs, err := r.GetDeploymentStableRs(canary)
 	if err != nil || canaryRs == nil {
 		return workload, err
 	}
@@ -422,7 +422,7 @@ func (r *ControllerFinder) GetReplicaSetsForDeployment(obj *apps.Deployment) ([]
 	return rss, nil
 }
 
-func (r *ControllerFinder) getDeploymentStableRs(obj *apps.Deployment) (*apps.ReplicaSet, error) {
+func (r *ControllerFinder) GetDeploymentStableRs(obj *apps.Deployment) (*apps.ReplicaSet, error) {
 	rss, err := r.GetReplicaSetsForDeployment(obj)
 	if err != nil {
 		return nil, err

pkg/webhook/util/writer/fs.go

@@ -128,7 +128,7 @@ func prepareToWrite(dir string) error {
 		// TODO: figure out if we can reduce the permission. (Now it's 0777)
 		err = os.MkdirAll(dir, 0777)
 		if err != nil {
-			return fmt.Errorf("can't create dir: %v", dir)
+			return fmt.Errorf("can't create dir: %v, err: %s", dir, err.Error())
 		}
 	case err != nil:
 		return err

pkg/webhook/workload/mutating/workload_update_handler.go

@@ -263,13 +263,13 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo
 	// in rollout progressing
 	if newObj.Annotations[util.InRolloutProgressingAnnotation] != "" {
 		modified := false
-		if !newObj.Spec.Paused {
-			modified = true
-			newObj.Spec.Paused = true
-		}
 		strategy := util.GetDeploymentStrategy(newObj)
-		switch strings.ToLower(string(strategy.RollingStyle)) {
+		// partition
-		case strings.ToLower(string(appsv1alpha1.PartitionRollingStyle)):
+		if strings.EqualFold(string(strategy.RollingStyle), string(appsv1alpha1.PartitionRollingStyle)) {
+			if !newObj.Spec.Paused {
+				modified = true
+				newObj.Spec.Paused = true
+			}
 			// Make sure it is always Recreate to disable native controller
 			if newObj.Spec.Strategy.Type == apps.RollingUpdateDeploymentStrategyType {
 				modified = true
@@ -287,7 +287,24 @@ func (h *WorkloadHandler) handleDeployment(newObj, oldObj *apps.Deployment) (boo
 			}
 			appsv1alpha1.SetDefaultDeploymentStrategy(&strategy)
 			setDeploymentStrategyAnnotation(strategy, newObj)
-		default:
+			// bluegreenStyle
+		} else if len(newObj.GetAnnotations()[appsv1beta1.OriginalDeploymentStrategyAnnotation]) > 0 {
+			if isEffectiveDeploymentRevisionChange(oldObj, newObj) {
+				newObj.Spec.Paused, modified = true, true
+				// disallow continuous release, allow rollback
+				klog.Warningf("rollback or continuous release detected in Deployment webhook, while only rollback is allowed for bluegreen release for now")
+			}
+			// not allow to modify Strategy.Type to Recreate
+			if newObj.Spec.Strategy.Type != apps.RollingUpdateDeploymentStrategyType {
+				modified = true
+				newObj.Spec.Strategy.Type = oldObj.Spec.Strategy.Type
+				klog.Warningf("Not allow to modify Strategy.Type to Recreate")
+			}
+		} else { // default
+			if !newObj.Spec.Paused {
+				modified = true
+				newObj.Spec.Paused = true
+			}
 			// Do not allow to modify strategy as Recreate during rolling
 			if newObj.Spec.Strategy.Type == apps.RecreateDeploymentStrategyType {
 				modified = true
@@ -369,6 +386,22 @@ func (h *WorkloadHandler) handleCloneSet(newObj, oldObj *kruiseappsv1alpha1.Clon
 	} else if rollout == nil || rollout.Spec.Strategy.IsEmptyRelease() {
 		return false, nil
 	}
+	/*
+		continuous release (or successive release) is not supported for bluegreen release, especially for cloneset,
+		here is why:
+		suppose we are releasing a cloneset, which has pods of both v1 and v2 for now. If we release v3 before
+		v2 release is done, the cloneset controller might scale down pods without distinguishing between v1 and v2.
+		This is because our implementation is based on the minReadySeconds, pods of both v1 and v2 are "unavailable"
+		in the progress of rollout.
+		Deployment actually has the same problem, however it is possible to bypass this issue for Deployment by setting
+		minReadySeconds for replicaset separately; unfortunately this workaround seems not work for cloneset
+
+		// if rollout.Spec.Strategy.IsBlueGreenRelease() && revision.IsContinuousRelease(h.Client, oldObj, newObj) {
+		// 	err = fmt.Errorf("successive release is not supported for bluegreen for now, rollback first")
+		// 	return false, fmt.Errorf(err.Error())
+		// }
+	*/
+
 	// if traffic routing, there must only be one version of Pods
 	if rollout.Spec.Strategy.HasTrafficRoutings() && newObj.Status.Replicas != newObj.Status.UpdatedReplicas {
 		klog.Warningf("Because cloneSet(%s/%s) have multiple versions of Pods, so can not enter rollout progressing", newObj.Namespace, newObj.Name)

pkg/webhook/workload/mutating/workload_update_handler_test.go

@@ -130,6 +130,51 @@ var (
 		},
 	}
 
+	rsDemoV2 = &apps.ReplicaSet{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "apps/v1",
+			Kind:       "ReplicaSet",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "echoserver-v2",
+			Labels: map[string]string{
+				"app":               "echoserver",
+				"pod-template-hash": "verision2",
+			},
+			Annotations: map[string]string{},
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(deploymentDemo, schema.GroupVersionKind{
+					Group:   apps.SchemeGroupVersion.Group,
+					Version: apps.SchemeGroupVersion.Version,
+					Kind:    "Deployment",
+				}),
+			},
+		},
+		Spec: apps.ReplicaSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"app": "echoserver",
+				},
+			},
+			Replicas: pointer.Int32(5),
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"app": "echoserver",
+					},
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:  "echoserver",
+							Image: "echoserver:v2",
+						},
+					},
+				},
+			},
+		},
+	}
+
 	cloneSetDemo = &kruisev1aplphal.CloneSet{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "apps.kruise.io/v1alpha1",
@@ -519,6 +564,100 @@ func TestHandlerDeployment(t *testing.T) {
 				return obj
 			},
 		},
+		{
+			name: "bluegreen: normal release",
+			getObjs: func() (*apps.Deployment, *apps.Deployment) {
+				oldObj := deploymentDemo.DeepCopy()
+				newObj := deploymentDemo.DeepCopy()
+				newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2"
+				return oldObj, newObj
+			},
+			expectObj: func() *apps.Deployment {
+				obj := deploymentDemo.DeepCopy()
+				obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2"
+				obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}`
+				obj.Spec.Paused = true
+				return obj
+			},
+			getRs: func() []*apps.ReplicaSet {
+				rs := rsDemo.DeepCopy()
+				return []*apps.ReplicaSet{rs}
+			},
+			getRollout: func() *appsv1beta1.Rollout {
+				obj := rolloutDemo.DeepCopy()
+				obj.Spec.Strategy.BlueGreen = &appsv1beta1.BlueGreenStrategy{}
+				return obj
+			},
+			isError: false,
+		},
+		{
+			name: "bluegreen: rollback",
+			getObjs: func() (*apps.Deployment, *apps.Deployment) {
+				oldObj := deploymentDemo.DeepCopy()
+				oldObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}`
+				oldObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}`
+				oldObj.Labels[appsv1alpha1.DeploymentStableRevisionLabel] = "5b494f7bf"
+				oldObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2"
+				newObj := deploymentDemo.DeepCopy()
+				newObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}`
+				newObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}`
+				newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v1"
+				return oldObj, newObj
+			},
+			expectObj: func() *apps.Deployment {
+				obj := deploymentDemo.DeepCopy()
+				obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v1"
+				obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}`
+				obj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}`
+				obj.Spec.Paused = true
+				return obj
+			},
+			getRs: func() []*apps.ReplicaSet {
+				rs := rsDemo.DeepCopy()
+				rs2 := rsDemoV2.DeepCopy()
+				return []*apps.ReplicaSet{rs, rs2}
+			},
+			getRollout: func() *appsv1beta1.Rollout {
+				obj := rolloutDemo.DeepCopy()
+				obj.Spec.Strategy.BlueGreen = &appsv1beta1.BlueGreenStrategy{}
+				return obj
+			},
+			isError: false,
+		},
+		{
+			name: "bluegreen: successive release",
+			getObjs: func() (*apps.Deployment, *apps.Deployment) {
+				oldObj := deploymentDemo.DeepCopy()
+				oldObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}`
+				oldObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}`
+				oldObj.Labels[appsv1alpha1.DeploymentStableRevisionLabel] = "5b494f7bf"
+				oldObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v2"
+				newObj := deploymentDemo.DeepCopy()
+				newObj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}`
+				newObj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}`
+				newObj.Spec.Template.Spec.Containers[0].Image = "echoserver:v3"
+				return oldObj, newObj
+			},
+			expectObj: func() *apps.Deployment {
+				obj := deploymentDemo.DeepCopy()
+				obj.Spec.Template.Spec.Containers[0].Image = "echoserver:v3"
+				obj.Annotations[util.InRolloutProgressingAnnotation] = `{"rolloutName":"rollout-demo"}`
+				obj.Annotations[appsv1beta1.OriginalDeploymentStrategyAnnotation] = `{"MaxSurge":"25%", "MaxUnavailable":"25%"}`
+				obj.Spec.Paused = true
+				return obj
+			},
+			getRs: func() []*apps.ReplicaSet {
+				rs := rsDemo.DeepCopy()
+				rs2 := rsDemoV2.DeepCopy()
+				return []*apps.ReplicaSet{rs, rs2}
+			},
+			getRollout: func() *appsv1beta1.Rollout {
+				obj := rolloutDemo.DeepCopy()
+				obj.Spec.Strategy.BlueGreen = &appsv1beta1.BlueGreenStrategy{}
+				return obj
+			},
+			isError: false,
+		},
 	}
 
 	decoder, _ := admission.NewDecoder(scheme)
@@ -542,8 +681,11 @@ func TestHandlerDeployment(t *testing.T) {
 
 			oldObj, newObj := cs.getObjs()
 			_, err := h.handleDeployment(newObj, oldObj)
-			if cs.isError && err == nil {
+			if cs.isError {
-				t.Fatal("handlerDeployment failed")
+				if err == nil {
+					t.Fatal("handlerDeployment failed")
+				}
+				return //no need to check again
 			} else if !cs.isError && err != nil {
 				t.Fatalf(err.Error())
 			}

test/e2e/rollout_v1beta1_test.go

@@ -112,25 +112,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 
 		return clone
 	}
-	// continuous release is not allowed for now, therefor we expect failure when updating
-	UpdateDeploymentFailed := func(object *apps.Deployment) *apps.Deployment {
-		var clone *apps.Deployment
-		Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error {
-			clone = &apps.Deployment{}
-			err := GetObject(object.Name, clone)
-			if err != nil {
-				return err
-			}
-			clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas)
-			clone.Spec.Template = *object.Spec.Template.DeepCopy()
-			clone.Labels = mergeMap(clone.Labels, object.Labels)
-			clone.Annotations = mergeMap(clone.Annotations, object.Annotations)
-			clone.Spec.Paused = object.Spec.Paused
-			return k8sClient.Update(context.TODO(), clone)
-		})).To(HaveOccurred())
-
-		return clone
-	}
 
 	UpdateCloneSet := func(object *appsv1alpha1.CloneSet) *appsv1alpha1.CloneSet {
 		var clone *appsv1alpha1.CloneSet
@@ -150,25 +131,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 		return clone
 	}
 
-	// continuous release is not allowed for now, therefor we expect failure when updating
-	UpdateCloneSetFail := func(object *appsv1alpha1.CloneSet) *appsv1alpha1.CloneSet {
-		var clone *appsv1alpha1.CloneSet
-		Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error {
-			clone = &appsv1alpha1.CloneSet{}
-			err := GetObject(object.Name, clone)
-			if err != nil {
-				return err
-			}
-			clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas)
-			clone.Spec.Template = *object.Spec.Template.DeepCopy()
-			clone.Labels = mergeMap(clone.Labels, object.Labels)
-			clone.Annotations = mergeMap(clone.Annotations, object.Annotations)
-			return k8sClient.Update(context.TODO(), clone)
-		})).To(HaveOccurred())
-
-		return clone
-	}
-
 	// UpdateDaemonSet := func(object *appsv1alpha1.DaemonSet) *appsv1alpha1.DaemonSet {
 	// 	var daemon *appsv1alpha1.DaemonSet
 	// 	Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error {
@@ -353,8 +315,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 		Eventually(func() bool {
 			clone := &apps.Deployment{}
 			Expect(GetObject(deployment.Name, clone)).NotTo(HaveOccurred())
-			return clone.Status.ObservedGeneration == clone.Generation &&
+			return clone.Status.ObservedGeneration == clone.Generation && clone.Status.ReadyReplicas == clone.Status.Replicas
-				*clone.Spec.Replicas == clone.Status.AvailableReplicas && clone.Status.ReadyReplicas == clone.Status.Replicas
 		}, 10*time.Minute, time.Second).Should(BeTrue())
 	}
 
@@ -1696,8 +1657,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status & paused
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
 			setting, _ := control.GetOriginalSetting(workload)
@@ -1751,8 +1711,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10))
 			Expect(workload.Spec.Paused).Should(BeFalse())
 			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
@@ -1777,8 +1736,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10))
 			Expect(workload.Spec.Paused).Should(BeFalse())
 			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
@@ -1817,8 +1775,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10))
 			Expect(workload.Spec.Paused).Should(BeFalse())
 			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
@@ -1868,7 +1825,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			for _, env := range workload.Spec.Template.Spec.Containers[0].Env {
 				if env.Name == "NODE_NAME" {
 					Expect(env.Value).Should(Equal("version2"))
@@ -1946,8 +1902,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status & paused
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
 			setting, _ := control.GetOriginalSetting(workload)
@@ -2001,8 +1956,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10))
 			Expect(workload.Spec.Paused).Should(BeFalse())
 			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
@@ -2027,8 +1981,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10))
 			Expect(workload.Spec.Paused).Should(BeFalse())
 			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
@@ -2067,8 +2020,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10))
 			Expect(workload.Spec.Paused).Should(BeFalse())
 			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
@@ -2105,8 +2057,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10))
 			Expect(workload.Spec.Paused).Should(BeFalse())
 			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
@@ -2143,8 +2094,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 10))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10))
 			Expect(workload.Spec.Paused).Should(BeFalse())
 			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
@@ -2246,8 +2196,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status & paused
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
 			setting, _ := control.GetOriginalSetting(workload)
@@ -2266,169 +2215,82 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// ----- Continuous Release ------
 			updatedRevision := rollout.Status.BlueGreenStatus.UpdatedRevision
 			By(updatedRevision)
-			By("update workload env NODE_NAME from(version2) -> to(version3)")
 			newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"})
 			workload.Spec.Template.Spec.Containers[0].Env = newEnvs
-			UpdateDeploymentFailed(workload)
+			UpdateDeployment(workload)
-			// the next code is used to test continuous release scenario, in case we need it in the future, keep it as comment
+			By("update workload env NODE_NAME from(version2) -> to(version3)")
-			/*
+			WaitRolloutStepPaused(rollout.Name, 1)
-				UpdateDeployment(workload)
+			stableRevision = GetStableRSRevision(workload)
-				// from step 1 to step 1, we need to additionally check stepUpgrad to distinguish the two steps
+			By(stableRevision)
-				WaitRolloutStepUpgrade(rollout.Name, 1)
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
-				WaitRolloutStepPaused(rollout.Name, 1)
+			Expect(rollout.Status.CanaryStatus).Should(BeNil())
-				// stable revision shouldn't change
+			Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision))
-				Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
-				Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
-				Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision))
-				Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).ShouldNot(Equal(updatedRevision))
-				Expect(workload.Labels[v1beta1.DeploymentStableRevisionLabel]).Should(Equal(stableRevision))
 
-				// check workload status & paused
+			// check workload status & paused
-				Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
+			time.Sleep(time.Second * 1) // ensure the Deployment controller notice the update
-				Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
-				Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) // no version3 pods created, since we don't support continuous release yet
-				Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
+			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
-				setting = control.GetOriginalSetting(workload)
+			setting, _ = control.GetOriginalSetting(workload)
-				Expect(setting.MinReadySeconds).Should(BeNumerically("==", int32(0)))
+			Expect(setting.MinReadySeconds).Should(BeNumerically("==", int32(0)))
-				Expect(*setting.ProgressDeadlineSeconds).Should(BeNumerically("==", int32(600)))
+			Expect(*setting.ProgressDeadlineSeconds).Should(BeNumerically("==", int32(600)))
-				Expect(reflect.DeepEqual(setting.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue())
+			Expect(reflect.DeepEqual(setting.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue())
-				Expect(reflect.DeepEqual(setting.MaxSurge, &intstr.IntOrString{Type: intstr.Int, IntVal: 1})).Should(BeTrue())
+			Expect(reflect.DeepEqual(setting.MaxSurge, &intstr.IntOrString{Type: intstr.Int, IntVal: 1})).Should(BeTrue())
 
-				Expect(workload.Spec.Paused).Should(BeFalse())
+			Expect(workload.Spec.Paused).Should(BeTrue()) // paused in the webhook
-				Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
+			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
-				Expect(workload.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds)))
+			Expect(workload.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds)))
-				Expect(*workload.Spec.ProgressDeadlineSeconds).Should(Equal(int32(v1beta1.MaxProgressSeconds)))
+			Expect(*workload.Spec.ProgressDeadlineSeconds).Should(Equal(int32(v1beta1.MaxProgressSeconds)))
-				Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue())
+			Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue())
-				Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxSurge, &intstr.IntOrString{Type: intstr.String, StrVal: "50%"})).Should(BeTrue())
+			Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxSurge, &intstr.IntOrString{Type: intstr.String, StrVal: "50%"})).Should(BeTrue())
 
-				// check rollout status
+			// it's ok to patch the Deployment to version2 back, and even release remaining steps then
-				Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"})
-				Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			workload.Spec.Template.Spec.Containers[0].Env = newEnvs
-				Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing))
+			UpdateDeployment(workload)
-				Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision))
+			By("update workload env NODE_NAME from(version3) -> to(version2)")
-				Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).Should(Equal(util.ComputeHash(&workload.Spec.Template, nil)))
+			WaitRolloutStepPaused(rollout.Name, 1)
-				Expect(rollout.Status.BlueGreenStatus.PodTemplateHash).Should(Equal(GetCanaryRSRevision(workload)))
+			stableRevision = GetStableRSRevision(workload)
-				canaryRevision := rollout.Status.BlueGreenStatus.PodTemplateHash
+			By(stableRevision)
-				Expect(rollout.Status.BlueGreenStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation]))
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
-				Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 1))
+			Expect(rollout.Status.CanaryStatus).Should(BeNil())
-				Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 2))
+			Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision))
-				Expect(rollout.Status.BlueGreenStatus.UpdatedReplicas).Should(BeNumerically("==", 3))
-				Expect(rollout.Status.BlueGreenStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3))
-				// check stable, canary service & ingress
-				// stable service
-				Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
-				Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision))
-				//canary service
-				cService := &v1.Service{}
-				Expect(GetObject(service.Name+"-canary", cService)).NotTo(HaveOccurred())
-				Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision))
 
-				// ------ step 2: replicas: 100%, traffic: 0% ------
+			// check workload status & paused
-				// resume rollout canary
+			time.Sleep(time.Second * 1) // ensure the Deployment controller notice the update
-				ResumeRollout(rollout.Name)
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
-				By("resume rollout, and wait next step(2)")
+			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-				WaitRolloutStepPaused(rollout.Name, 2)
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
+			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
-				// workload
+			setting, _ = control.GetOriginalSetting(workload)
-				Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			Expect(setting.MinReadySeconds).Should(BeNumerically("==", int32(0)))
-				Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
+			Expect(*setting.ProgressDeadlineSeconds).Should(BeNumerically("==", int32(600)))
-				Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 5))
+			Expect(reflect.DeepEqual(setting.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue())
-				Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
+			Expect(reflect.DeepEqual(setting.MaxSurge, &intstr.IntOrString{Type: intstr.Int, IntVal: 1})).Should(BeTrue())
-				Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 10))
-				Expect(workload.Spec.Paused).Should(BeFalse())
-				Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
-				Expect(workload.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds)))
-				Expect(*workload.Spec.ProgressDeadlineSeconds).Should(Equal(int32(v1beta1.MaxProgressSeconds)))
-				Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue())
-				Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxSurge, &intstr.IntOrString{Type: intstr.String, StrVal: "100%"})).Should(BeTrue())
 
-				// rollout
+			Expect(workload.Spec.Paused).Should(BeTrue())
-				Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
-				Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 2))
+			Expect(workload.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds)))
-				Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 3))
+			Expect(*workload.Spec.ProgressDeadlineSeconds).Should(Equal(int32(v1beta1.MaxProgressSeconds)))
-				Expect(rollout.Status.BlueGreenStatus.UpdatedReplicas).Should(BeNumerically("==", 5))
+			Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue())
-				Expect(rollout.Status.BlueGreenStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 5))
+			Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxSurge, &intstr.IntOrString{Type: intstr.String, StrVal: "50%"})).Should(BeTrue())
-				// ----- Continuous Release, AGAIN------
-				updatedRevision = rollout.Status.BlueGreenStatus.UpdatedRevision
-				By("update workload env NODE_NAME from(version3) -> to(version4)")
-				Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
-				newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version4"})
-				workload.Spec.Template.Spec.Containers[0].Env = newEnvs
-				UpdateDeployment(workload)
-				WaitRolloutStepPaused(rollout.Name, 1)
-				// stable revision shouldn't change
-				Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
-				Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
-				Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision))
-				Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).ShouldNot(Equal(updatedRevision))
-				Expect(workload.Labels[v1beta1.DeploymentStableRevisionLabel]).Should(Equal(stableRevision))
-				// workload
-				Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
-				Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-				Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
-				Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
-				Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
-				Expect(workload.Spec.Paused).Should(BeFalse())
-				Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
-				Expect(workload.Spec.MinReadySeconds).Should(Equal(int32(v1beta1.MaxReadySeconds)))
-				Expect(*workload.Spec.ProgressDeadlineSeconds).Should(Equal(int32(v1beta1.MaxProgressSeconds)))
-				Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxUnavailable, &intstr.IntOrString{Type: intstr.Int, IntVal: 0})).Should(BeTrue())
-				Expect(reflect.DeepEqual(workload.Spec.Strategy.RollingUpdate.MaxSurge, &intstr.IntOrString{Type: intstr.String, StrVal: "50%"})).Should(BeTrue())
 
-				// rollout
+			// of course user can rollback to version1 directly
-				Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"})
-				Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 1))
+			workload.Spec.Template.Spec.Containers[0].Env = newEnvs
-				Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 2))
+			UpdateDeployment(workload)
-				Expect(rollout.Status.BlueGreenStatus.UpdatedReplicas).Should(BeNumerically("==", 3))
+			By("rollback: update workload env NODE_NAME from(version2) -> to(version1)")
-				Expect(rollout.Status.BlueGreenStatus.UpdatedReadyReplicas).Should(BeNumerically("==", 3))
+			WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy)
+			WaitDeploymentAllPodsReady(workload)
 
-				// ------ step 4: replicas: 100%, traffic: 100% ------
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
-				// resume rollout canary
+			Expect(len(rollout.GetAnnotations()[v1beta1.OriginalDeploymentStrategyAnnotation])).Should(BeNumerically("==", 0)) // the annotation should be removed
-				By("Jump to step 4")
+			cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing)
-				JumpRolloutStep(rollout.Name, 4)
+			Expect(string(cond.Reason)).Should(Equal(string(v1beta1.CanaryStepStateCompleted)))
-				WaitRolloutStepPaused(rollout.Name, 4)
+			Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse)))
-
-				// ------ Final approval ------
-				// resume rollout canary
-				ResumeRollout(rollout.Name)
-				By("resume rollout, final approval")
-				// wait rollout complete
-				WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhase(v1beta1.RolloutPhaseHealthy))
-				klog.Infof("rollout(%s) completed, and check", namespace)
-				// rollout
-				Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
-				Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", -1))
-				// check service & ingress
-				// ingress
-				Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred())
-				cIngress := &netv1.Ingress{}
-				Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred())
-				// service
-				Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
-				Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(""))
-				cService = &v1.Service{}
-				Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred())
-				// workload
-				Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
-				Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
-				Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas))
-				Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
-				for _, env := range workload.Spec.Template.Spec.Containers[0].Env {
-					if env.Name == "NODE_NAME" {
-						Expect(env.Value).Should(Equal("version4"))
-					}
-				}
-				// check progressing succeed
-				Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
-				cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing)
-				Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted))
-				Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse)))
-				cond = getRolloutCondition(rollout.Status, v1beta1.RolloutConditionSucceeded)
-				Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue)))
-				Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
-				WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation)
-			*/
 		})
 
 		It("bluegreen scale up and down", func() {
@@ -2480,8 +2342,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status & paused
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
 			// check rollout status
@@ -2496,6 +2357,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// ------ 50% maxSurge, scale up: from 5 to 6 ------
 			workload.Spec.Replicas = utilpointer.Int32(6)
 			UpdateDeployment(workload)
+			By("scale up: from 5 to 6")
 			time.Sleep(time.Second * 3)
 			WaitDeploymentBlueGreenReplicas(workload)
 
@@ -2508,8 +2370,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 9))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 6))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 9))
 
 			// ------ scale up: from 6 to 7 ------
@@ -2524,13 +2385,13 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 4))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 11))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 7))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 11))
 
 			// ------ scale up: from 7 to 8 ------
 			workload.Spec.Replicas = utilpointer.Int32(8)
 			UpdateDeployment(workload)
+			By("scale up: from 7 to 8")
 			time.Sleep(time.Second * 3)
 			WaitDeploymentBlueGreenReplicas(workload)
 			// check rollout status
@@ -2540,13 +2401,13 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 4))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 12))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 8))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 12))
 
 			// ------ scale down: from 8 to 4 ------
 			workload.Spec.Replicas = utilpointer.Int32(4)
 			UpdateDeployment(workload)
+			By("scale down: from 8 to 4")
 			time.Sleep(time.Second * 3)
 			WaitDeploymentBlueGreenReplicas(workload)
 			// check rollout status
@@ -2556,8 +2417,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 2))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 2))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 6))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 4))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 6))
 
 			// ------ step 2: replicas: 100%, traffic: 0% ------
@@ -2569,8 +2429,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 4))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 4))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 			Expect(workload.Spec.Paused).Should(BeFalse())
 			Expect(workload.Spec.Strategy.Type).Should(Equal(apps.RollingUpdateDeploymentStrategyType))
@@ -2598,8 +2457,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 7))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 7))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 14))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 7))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 14))
 
 			// ------ scale up: from 7 to 8 ------
@@ -2614,8 +2472,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 16))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 8))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 16))
 
 			// ------ scale down: from 8 to 4 ------
@@ -2630,8 +2487,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 4))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 4))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
 		})
@@ -2685,8 +2541,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status & paused
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
 			// check rollout status
@@ -2732,7 +2587,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 		})
 
 		It("bluegreen disable rollout case", func() {
@@ -2784,8 +2638,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status & paused
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
 			// check rollout status
@@ -2846,7 +2699,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 		})
 	})
 
@@ -2906,8 +2758,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status & paused
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
 			// check rollout status
@@ -2959,7 +2810,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 
 			// check hpa
 			Expect(GetObject(hpa.Name, hpa)).NotTo(HaveOccurred())
@@ -3021,8 +2871,7 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// check workload status & paused
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3))
-			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 3))
+			Expect(workload.Status.UnavailableReplicas).Should(BeNumerically("==", 8))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
 			// check rollout status
@@ -3073,7 +2922,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// status
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			// check hpa
 			Expect(GetObject(hpa.Name, hpa)).NotTo(HaveOccurred())
 			Expect(hpa.Spec.ScaleTargetRef.Name).Should(Equal(workload.Name))
@@ -3225,7 +3073,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// workload
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			for _, env := range workload.Spec.Template.Spec.Containers[0].Env {
 				if env.Name == "NODE_NAME" {
 					Expect(env.Value).Should(Equal("version2"))
@@ -3525,12 +3372,13 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
 			Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing))
 			Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision))
+			revision2 := workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:]
 			Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:]))
 			Expect(rollout.Status.BlueGreenStatus.PodTemplateHash).Should(Equal(workload.Status.UpdateRevision[strings.LastIndex(workload.Status.UpdateRevision, "-")+1:]))
 			Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 2))
 			Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 3))
 			Expect(rollout.Status.BlueGreenStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation]))
-			// if network configuration has restored
+			// check if network configuration has restored
 			cIngress := &netv1.Ingress{}
 			Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred())
 			Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true"))
@@ -3542,7 +3390,77 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			By("update workload env NODE_NAME from(version2) -> to(version3)")
 			newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"})
 			workload.Spec.Template.Spec.Containers[0].Env = newEnvs
-			UpdateCloneSetFail(workload)
+			UpdateCloneSet(workload)
+			time.Sleep(time.Second * 1)
+			WaitRolloutStepPaused(rollout.Name, 2)
+			// check workload status & paused
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			Expect(workload.Status.Replicas).Should(BeNumerically("==", 10))
+			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0))
+			Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 0))
+			Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) // unlike Deployment, cloneSet isn't paused
+			By("check cloneSet status & paused success")
+			// check rollout status
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing))
+			Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision))
+			/*
+				note: rollout.Status.BlueGreenStatus.UpdatedRevision won't update at all, since we disallow
+				continuous release for bluegreen release (it is designed to trigger a fatal error before status update)
+				however the workload.Status.UpdateRevision will always be update since it is calculated
+				directly from the Cloneset
+			*/
+			Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).Should(Equal(revision2))
+			Expect(rollout.Status.BlueGreenStatus.PodTemplateHash).Should(Equal(revision2))
+			Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 2))
+			Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 3))
+			Expect(rollout.Status.BlueGreenStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation]))
+
+			// it's ok to patch the CloneSet to version2 back, and even release remaining steps then
+			newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"})
+			workload.Spec.Template.Spec.Containers[0].Env = newEnvs
+			UpdateCloneSet(workload)
+			By("update workload env NODE_NAME from(version3) -> to(version2)")
+			time.Sleep(time.Second * 1)
+			WaitRolloutStepPaused(rollout.Name, 2)
+			// check workload status & paused
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			Expect(workload.Status.Replicas).Should(BeNumerically("==", 10))
+			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) // unlike Deployment, cloneSet isn't paused
+			By("check cloneSet status & paused success")
+			// check rollout status
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseProgressing))
+			Expect(rollout.Status.BlueGreenStatus.StableRevision).Should(Equal(stableRevision))
+			Expect(rollout.Status.BlueGreenStatus.UpdatedRevision).Should(Equal(revision2))
+			Expect(rollout.Status.BlueGreenStatus.PodTemplateHash).Should(Equal(revision2))
+			Expect(rollout.Status.BlueGreenStatus.CurrentStepIndex).Should(BeNumerically("==", 2))
+			Expect(rollout.Status.BlueGreenStatus.NextStepIndex).Should(BeNumerically("==", 3))
+			Expect(rollout.Status.BlueGreenStatus.RolloutHash).Should(Equal(rollout.Annotations[util.RolloutHashAnnotation]))
+
+			// of course user can rollback to version1 directly
+			newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"})
+			workload.Spec.Template.Spec.Containers[0].Env = newEnvs
+			UpdateCloneSet(workload)
+			By("rollback: update workload env NODE_NAME from(version2) -> to(version1)")
+			WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy)
+			WaitCloneSetAllPodsReady(workload)
+
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			cond := getRolloutCondition(rollout.Status, v1beta1.RolloutConditionProgressing)
+			Expect(string(cond.Reason)).Should(Equal(string(v1beta1.CanaryStepStateCompleted)))
+			Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse)))
+			Expect(len(rollout.GetAnnotations()[v1beta1.OriginalDeploymentStrategyAnnotation])).Should(BeNumerically("==", 0)) // the annotation should be removed
+			CheckIngressRestored(service.Name)
+
+			// check workload status & paused
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			Expect(workload.Status.Replicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5))
+			Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse())
 		})
 
 		// cloneset now only support single step, keep this case for future
@@ -3631,7 +3549,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 		// 	// check workload status
 		// 	Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 		// 	Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 6))
-		// 	Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 6))
 		// 	Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 12))
 
 		// 	// ------ scale up: from 6 to 7 ------
@@ -3650,7 +3567,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 		// 	// check workload status
 		// 	Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 		// 	Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 7))
-		// 	Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 7))
 		// 	Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 14))
 
 		// 	// ------ scale up: from 7 to 8 ------
@@ -3669,7 +3585,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 		// 	// check workload status
 		// 	Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 		// 	Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 8))
-		// 	Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 8))
 		// 	Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 16))
 
 		// 	// ------ scale down: from 8 to 4 ------
@@ -3688,7 +3603,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 		// 	// check workload status
 		// 	Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 		// 	Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 4))
-		// 	Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 4))
 		// 	Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", 8))
 
 		// })
@@ -3808,7 +3722,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 				}
 			}
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 
 			// check service & ingress & deployment
 			// ingress
@@ -3946,7 +3859,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 				}
 			}
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 
 			// check service & ingress & deployment
 			// ingress
@@ -4638,7 +4550,6 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
 			// cloneset
 			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
 			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5))
-			Expect(workload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
 			for _, env := range workload.Spec.Template.Spec.Containers[0].Env {
 				if env.Name == "NODE_NAME" {
 					Expect(env.Value).Should(Equal("version2"))