The rollout controller code is basically complete

Dockerfile

@@ -5,14 +5,16 @@ WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download
+
+# Copy the dependencies
+COPY vendor/ vendor/
 
 # Copy the go source
 COPY main.go main.go
 COPY api/ api/
 COPY controllers/ controllers/
+COPY pkg/ pkg/
+COPY webhook/ webhook/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go

Makefile

@@ -63,7 +63,7 @@ build: generate fmt vet ## Build manager binary.
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
-docker-build: test ## Build docker image with the manager.
+docker-build: ## Build docker image with the manager.
 	docker build -t ${IMG} .
 
 docker-push: ## Push docker image with the manager.

PROJECT

@@ -15,6 +15,7 @@ resources:
 - api:
     crdVersion: v1
     namespaced: true
+  controller: true
   domain: kruise.io
   group: rollouts
   kind: BatchRelease

api/v1alpha1/batchrelease_plan_types.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2022 The Kruise & KubeVela Authors.
+Copyright 2022 The Kruise Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,43 +24,32 @@ import (
 // ReleaseStrategyType defines strategies for pods rollout
 type ReleaseStrategyType string
 
-type ReleasingBatchStateType string
-
-const (
-	InitializeBatchState ReleasingBatchStateType = "InitializeInBatch"
-	DoCanaryBatchState   ReleasingBatchStateType = "DoCanaryInBatch"
-	VerifyBatchState     ReleasingBatchStateType = "VerifyInBatch"
-	ReadyBatchState      ReleasingBatchStateType = "ReadyInBatch"
-)
-
-const (
-	// RolloutPhaseVerify indicates the phase of verifying workload health.
-	RolloutPhaseVerify RolloutPhase = "Verifying"
-	// RolloutPhaseFinalizing indicates the phase, where controller will do some operation after all batch ready.
-	RolloutPhaseFinalizing RolloutPhase = "Finalizing"
-	// RolloutPhaseCompleted indicates the release plan was executed successfully.
-	RolloutPhaseCompleted RolloutPhase = "Completed"
-)
-
-// ReleasePlan fines the details of the release plan
+// ReleasePlan fines the details of the rollout plan
 type ReleasePlan struct {
-	// Batches describe the plan for each batch in detail, including canary replicas and paused seconds.
+	// The exact distribution among batches.
+	// its size has to be exactly the same as the NumBatches (if set)
+	// The total number cannot exceed the targetSize or the size of the source resource
+	// We will IGNORE the last batch's replica field if it's a percentage since round errors can lead to inaccurate sum
+	// We highly recommend to leave the last batch's replica field empty
+	// +optional
 	Batches []ReleaseBatch `json:"batches"`
 	// All pods in the batches up to the batchPartition (included) will have
-	// the target updated specification while the rest still have the stable resource
-	// This is designed for the operators to manually release plan
-	// Default is nil, which means no partition.
+	// the target resource specification while the rest still have the source resource
+	// This is designed for the operators to manually rollout
+	// Default is the the number of batches which will rollout all the batches
 	// +optional
 	BatchPartition *int32 `json:"batchPartition,omitempty"`
-	// Paused the release plan executor, default is false
+	// Paused the rollout, default is false
 	// +optional
 	Paused bool `json:"paused,omitempty"`
 }
 
-// ReleaseBatch is used to describe how each release batch should be
+// ReleaseBatch is used to describe how the each batch rollout should be
 type ReleaseBatch struct {
-	// Replicas is the number of expected canary pods in this batch
-	// it can be an absolute number (ex: 5) or a percentage of total pods.
+	// Replicas is the number of pods to upgrade in this batch
+	// it can be an absolute number (ex: 5) or a percentage of total pods
+	// we will ignore the percentage of the last batch to just fill the gap
+	// +optional
 	// it is mutually exclusive with the PodList field
 	CanaryReplicas intstr.IntOrString `json:"canaryReplicas"`
 	// The wait time, in seconds, between instances upgrades, default = 0
@@ -70,9 +59,9 @@ type ReleaseBatch struct {
 
 // BatchReleaseStatus defines the observed state of a rollout plan
 type BatchReleaseStatus struct {
-	// Conditions records some important message at each Phase.
+	// Conditions represents the latest available observations of a CloneSet's current state.
 	Conditions []RolloutCondition `json:"conditions,omitempty"`
-	// Canary describes the status of the canary
+	// Canary describes the state of the canary rollout
 	CanaryStatus BatchReleaseCanaryStatus `json:"canaryStatus,omitempty"`
 	// StableRevision is the pod-template-hash of stable revision pod.
 	StableRevision string `json:"stableRevision,omitempty"`
@@ -84,6 +73,11 @@ type BatchReleaseStatus struct {
 	// ObservedWorkloadReplicas is the size of the target resources. This is determined once the initial spec verification
 	// and does not change until the rollout is restarted.
 	ObservedWorkloadReplicas int32 `json:"observedWorkloadReplicas,omitempty"`
+	// Count of hash collisions for creating canary Deployment. The controller uses this
+	// field as a collision avoidance mechanism when it needs to create the name for the
+	// newest canary Deployment.
+	// +optional
+	CollisionCount *int32 `json:"collisionCount,omitempty"`
 	// ObservedReleasePlanHash is a hash code of observed itself releasePlan.Batches.
 	ObservedReleasePlanHash string `json:"observedReleasePlanHash,omitempty"`
 	// Phase is the release phase.
@@ -92,8 +86,8 @@ type BatchReleaseStatus struct {
 }
 
 type BatchReleaseCanaryStatus struct {
-	// State indicates the state of the current batch.
-	State ReleasingBatchStateType `json:"state,omitempty"`
+	// ReleasingBatchState indicates the state of the current batch.
+	ReleasingBatchState ReleasingBatchStateType `json:"batchState,omitempty"`
 	// The current batch the rollout is working on/blocked, it starts from 0
 	CurrentBatch int32 `json:"currentBatch"`
 	// LastBatchFinalizedTime is the timestamp of
@@ -103,3 +97,12 @@ type BatchReleaseCanaryStatus struct {
 	// UpgradedReadyReplicas is the number of Pods upgraded by the rollout controller that have a Ready Condition.
 	UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"`
 }
+
+type ReleasingBatchStateType string
+
+const (
+	InitializeBatchState ReleasingBatchStateType = "InitializeInBatch"
+	DoCanaryBatchState   ReleasingBatchStateType = "DoCanaryInBatch"
+	VerifyBatchState     ReleasingBatchStateType = "VerifyInBatch"
+	ReadyBatchState      ReleasingBatchStateType = "ReadyInBatch"
+)

api/v1alpha1/batchrelease_types.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2022 The Kruise & KubeVela Authors.
+Copyright 2022 The Kruise Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -38,17 +38,28 @@ type BatchRelease struct {
 	Status BatchReleaseStatus `json:"status,omitempty"`
 }
 
-// BatchReleaseSpec defines how to describe a batch release plan.
+// BatchReleaseSpec defines how to describe an update between different compRevision
 type BatchReleaseSpec struct {
-	// Cancelled is true indicates this batch release plan is cancelled.
-	// All resources about canary will be cleaned up.
+	Strategy ReleaseStrategy `json:"strategy,omitempty"`
+	// Cancel is true indicates this batch release plan is cancelled.
 	Cancelled bool `json:"cancelled,omitempty"`
-	// TargetRef contains the name of the workload that we need to upgrade to.
+	// TargetRevisionName contains the name of the componentRevisionName that we need to upgrade to.
 	TargetRef ObjectRef `json:"targetReference"`
-	// ReleasePlan is the details on how to release the updated revision.
+	// RolloutPlan is the details on how to rollout the resources
 	ReleasePlan ReleasePlan `json:"releasePlan"`
 }
 
+type ReleaseStrategy struct {
+	// +optional
+	CloneSetStrategy CloneSetReleaseStrategyType `json:"cloneSetStrategy,omitempty"`
+	// +optional
+	DeploymentStrategy DeploymentReleaseStrategyType `json:"deploymentStrategy,omitempty"`
+}
+
+type CloneSetReleaseStrategyType string
+
+type DeploymentReleaseStrategyType string
+
 // BatchReleaseList contains a list of BatchRelease
 // +kubebuilder:object:root=true
 type BatchReleaseList struct {

api/v1alpha1/groupversion_info.go

@@ -16,7 +16,7 @@ limitations under the License.
 
 // Package v1alpha1 contains API Schema definitions for the apps v1alpha1 API group
 //+kubebuilder:object:generate=true
-//+groupName=apps.rollouts.io
+//+groupName=rollouts.kruise.io
 package v1alpha1
 
 import (

api/v1alpha1/rollout_types.go

@@ -19,6 +19,7 @@ package v1alpha1
 import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
@@ -29,15 +30,34 @@ type RolloutSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
-	// TargetRef contains enough information to let you identify a workload for Rollout
-	TargetRef ObjectRef `json:"targetRef"`
+	ObjectRef ObjectRef `json:"objectRef"`
 
 	// The deployment strategy to use to replace existing pods with new ones.
 	Strategy RolloutStrategy `json:"strategy"`
 }
 
-// ObjectRef holds a references to the Kubernetes object
 type ObjectRef struct {
+	// workloadRef, revisionRef
+	// default is workloadRef
+	Type ObjectRefType `json:"type"`
+	// WorkloadRef contains enough information to let you identify a workload for Rollout
+	// Batch release of the bypass
+	WorkloadRef *WorkloadRef `json:"workloadRef,omitempty"`
+
+	// revisionRef
+	// Fully managed batch publishing capability
+	//RevisionRef *ControllerRevisionRef `json:"revisionRef,omitempty"`
+}
+
+type ObjectRefType string
+
+const (
+	WorkloadRefType = "workloadRef"
+	RevisionRefType = "revisionRef"
+)
+
+// WorkloadRef holds a references to the Kubernetes object
+type WorkloadRef struct {
 	// API Version of the referent
 	APIVersion string `json:"apiVersion"`
 	// Kind of the referent
@@ -46,28 +66,38 @@ type ObjectRef struct {
 	Name string `json:"name"`
 }
 
+type ControllerRevisionRef struct {
+	TargetRevisionName string `json:"targetRevisionName"`
+	SourceRevisionName string `json:"sourceRevisionName"`
+}
+
 // RolloutStrategy defines strategy to apply during next rollout
 type RolloutStrategy struct {
+	// Paused indicates that the Rollout is paused.
+	// Default value is false
+	Paused bool `json:"paused,omitempty"`
+	// canaryPlan, BlueGreenPlan
+	// Default value is canaryPlan
+	Type RolloutStrategyType `json:"type"`
 	// +optional
-	// BlueGreen *BlueGreenStrategy `json:"blueGreen,omitempty" protobuf:"bytes,1,opt,name=blueGreen"`
+	CanaryPlan *CanaryStrategy `json:"canaryPlan,omitempty"`
 	// +optional
-	Canary *CanaryStrategy `json:"canary,omitempty"`
+	// BlueGreenPlan *BlueGreenStrategy `json:"blueGreenPlan,omitempty"`
 }
 
+type RolloutStrategyType string
+
+const (
+	RolloutStrategyCanary RolloutStrategyType = "canaryPlan"
+)
+
 // CanaryStrategy defines parameters for a Replica Based Canary
 type CanaryStrategy struct {
-	// CanaryService holds the name of a service which selects pods with canary version and don't select any pods with stable version.
-	// +optional
-	//CanaryService string `json:"canaryService,omitempty"`
-	// StableService holds the name of a service which selects pods with stable version and don't select any pods with canary version.
-	// +optional
-	StableService string `json:"stableService,omitempty"`
 	// Steps define the order of phases to execute the canary deployment
 	// +optional
 	Steps []CanaryStep `json:"steps,omitempty"`
 	// TrafficRouting hosts all the supported service meshes supported to enable more fine-grained traffic routing
 	TrafficRouting *TrafficRouting `json:"trafficRouting,omitempty"`
-
 	// MetricsAnalysis runs a separate analysisRun while all the steps execute. This is intended to be a continuous validation of the new ReplicaSet
 	// MetricsAnalysis *MetricsAnalysisBackground `json:"metricsAnalysis,omitempty"`
 }
@@ -75,11 +105,15 @@ type CanaryStrategy struct {
 // CanaryStep defines a step of a canary workload.
 type CanaryStep struct {
 	// SetWeight sets what percentage of the canary pods should receive
-	SetWeight *int32 `json:"setWeight,omitempty"`
+	Weight int32 `json:"weight,omitempty"`
+	// Replicas is the number of expected canary pods in this batch
+	// it can be an absolute number (ex: 5) or a percentage of total pods.
+	// it is mutually exclusive with the PodList field
+	CanaryReplicas *intstr.IntOrString `json:"canaryReplicas,omitempty"`
 	// Pause freezes the rollout by setting spec.Paused to true.
 	// A Rollout will resume when spec.Paused is reset to false.
 	// +optional
-	Pause *RolloutPause `json:"pause,omitempty"`
+	Pause RolloutPause `json:"pause,omitempty"`
 	// MetricsAnalysis defines the AnalysisRun that will run for a step
 	// MetricsAnalysis *RolloutAnalysis `json:"metricsAnalysis,omitempty"`
 }
@@ -93,41 +127,25 @@ type RolloutPause struct {
 
 // TrafficRouting hosts all the different configuration for supported service meshes to enable more fine-grained traffic routing
 type TrafficRouting struct {
-	Type TrafficRoutingType `json:"type,omitempty"`
+	// Service holds the name of a service which selects pods with stable version and don't select any pods with canary version.
+	// +optional
+	Service string `json:"service"`
+	// Nginx, Alb, Istio etc.
+	Type TrafficRoutingType `json:"type"`
 	// Nginx holds Nginx Ingress specific configuration to route traffic
 	Nginx *NginxTrafficRouting `json:"nginx,omitempty"`
-	//
-	Alb *AlbTrafficRouting `json:"alb,omitempty"`
 }
 
 type TrafficRoutingType string
 
 const (
 	TrafficRoutingNginx TrafficRoutingType = "nginx"
-	TrafficRoutingAlb   TrafficRoutingType = "alb"
 )
 
 // NginxTrafficRouting configuration for Nginx ingress controller to control traffic routing
 type NginxTrafficRouting struct {
 	// Ingress refers to the name of an `Ingress` resource in the same namespace as the `Rollout`
 	Ingress string `json:"ingress"`
-	// A/B Testing
-	Tickets *Tickets `json:"tickets,omitempty"`
-}
-
-// AlbTrafficRouting configuration for Nginx ingress controller to control traffic routing
-type AlbTrafficRouting struct {
-	// Ingress refers to the name of an `Ingress` resource in the same namespace as the `Rollout`
-	Ingress string `json:"ingress"`
-	// A/B Testing
-	Tickets *Tickets `json:"tickets,omitempty"`
-}
-
-type Tickets struct {
-	// +optional
-	Header map[string]string `json:"header,omitempty"`
-	// +optional
-	Cookie map[string]string `json:"cookie,omitempty"`
 }
 
 // RolloutStatus defines the observed state of Rollout
@@ -138,9 +156,9 @@ type RolloutStatus struct {
 	// observedGeneration is the most recent generation observed for this SidecarSet. It corresponds to the
 	// SidecarSet's generation, which is updated on mutation by the API Server.
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
-	// UpdateRevision the hash of the current pod template
+	// CanaryRevision the hash of the canary pod template
 	// +optional
-	UpdateRevision string `json:"updateRevision,omitempty"`
+	CanaryRevision string `json:"canaryRevision,omitempty"`
 	// StableRevision indicates the revision pods that has successfully rolled out
 	StableRevision string `json:"stableRevision,omitempty"`
 	// Conditions a list of conditions a rollout can have.
@@ -180,18 +198,23 @@ const (
 	// considered when a new replica set is created or adopted, when pods scale
 	// up or old pods scale down, or when the services are updated. Progress is not estimated
 	// for paused rollouts.
-	RolloutConditionProgressing RolloutConditionType = "Progressing"
-	// reason
-	ProgressingReasonInitializing = "Initializing"
-	ProgressingReasonInRolling    = "InRolling"
-	ProgressingReasonFinalising   = "Finalising"
-	ProgressingReasonFailed       = "Failed"
-	ProgressingReasonSucceeded    = "Succeeded"
+	RolloutConditionProgressing   RolloutConditionType = "Progressing"
+	ProgressingReasonInitializing                      = "Initializing"
+	ProgressingReasonInRolling                         = "InRolling"
+	ProgressingReasonFinalising                        = "Finalising"
+	ProgressingReasonSucceeded                         = "Succeeded"
+	ProgressingReasonCanceled                          = "Canceled"
+	ProgressingReasonPaused                            = "Paused"
 
-	RolloutConditionTerminating RolloutConditionType = "Terminating"
-	//reason
-	TerminatingReasonInTerminating = "InTerminating"
-	TerminatingReasonCompleted     = "Completed"
+	// Terminating condition
+	RolloutConditionTerminating    RolloutConditionType = "Terminating"
+	TerminatingReasonInTerminating                      = "InTerminating"
+	TerminatingReasonCompleted                          = "Completed"
+
+	// Rollback condition
+	RolloutConditionRollback RolloutConditionType = "Rollback"
+	RollbackReasonInRollback                      = "InRollback"
+	RollbackReasonCompleted                       = "Completed"
 )
 
 // CanaryStatus status fields that only pertain to the canary rollout
@@ -211,7 +234,7 @@ type CanaryStatus struct {
 	CurrentStepState CanaryStepState `json:"currentStepState"`
 	Message          string          `json:"message,omitempty"`
 	// The last time this step pods is ready.
-	LastReadyTime *metav1.Time `json:"lastReadyTime,omitempty"`
+	LastUpdateTime *metav1.Time `json:"lastReadyTime,omitempty"`
 }
 
 type CanaryStepState string
@@ -230,14 +253,24 @@ type RolloutPhase string
 const (
 	// RolloutPhaseInitial indicates a rollout is Initial
 	RolloutPhaseInitial RolloutPhase = "Initial"
+	// RolloutPhaseVerify indicates a rollout is verifying
+	RolloutPhaseVerify RolloutPhase = "Verifying"
 	// RolloutPhaseHealthy indicates a rollout is healthy
 	RolloutPhaseHealthy RolloutPhase = "Healthy"
+	// RolloutPhasePreparing indicates a rollout is preparing for next progress.
+	RolloutPhasePreparing RolloutPhase = "Preparing"
 	// RolloutPhaseProgressing indicates a rollout is not yet healthy but still making progress towards a healthy state
 	RolloutPhaseProgressing RolloutPhase = "Progressing"
-	// RolloutPhasePaused indicates a rollout is not yet healthy and will not make progress until paused=false
-	RolloutPhasePaused RolloutPhase = "Paused"
+	// RolloutPhaseFinalizing indicates a rollout is finalizing
+	RolloutPhaseFinalizing RolloutPhase = "Finalizing"
 	// RolloutPhaseTerminating indicates a rollout is terminated
 	RolloutPhaseTerminating RolloutPhase = "Terminating"
+	// RolloutPhaseRollback indicates rollback
+	RolloutPhaseRollback RolloutPhase = "Rollback"
+	// RolloutPhaseCompleted indicates a rollout is completed
+	RolloutPhaseCompleted RolloutPhase = "Completed"
+	// RolloutPhaseCancelled indicates a rollout is cancelled
+	RolloutPhaseCancelled RolloutPhase = "Cancelled"
 )
 
 // +genclient

api/v1alpha1/zz_generated.deepcopy.go

@@ -22,28 +22,9 @@ package v1alpha1
 
 import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *AlbTrafficRouting) DeepCopyInto(out *AlbTrafficRouting) {
-	*out = *in
-	if in.Tickets != nil {
-		in, out := &in.Tickets, &out.Tickets
-		*out = new(Tickets)
-		(*in).DeepCopyInto(*out)
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AlbTrafficRouting.
-func (in *AlbTrafficRouting) DeepCopy() *AlbTrafficRouting {
-	if in == nil {
-		return nil
-	}
-	out := new(AlbTrafficRouting)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *BatchRelease) DeepCopyInto(out *BatchRelease) {
 	*out = *in
@@ -122,7 +103,8 @@ func (in *BatchReleaseList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *BatchReleaseSpec) DeepCopyInto(out *BatchReleaseSpec) {
 	*out = *in
-	out.TargetRef = in.TargetRef
+	out.Strategy = in.Strategy
+	in.TargetRef.DeepCopyInto(&out.TargetRef)
 	in.ReleasePlan.DeepCopyInto(&out.ReleasePlan)
 }
 
@@ -147,6 +129,11 @@ func (in *BatchReleaseStatus) DeepCopyInto(out *BatchReleaseStatus) {
 		}
 	}
 	in.CanaryStatus.DeepCopyInto(&out.CanaryStatus)
+	if in.CollisionCount != nil {
+		in, out := &in.CollisionCount, &out.CollisionCount
+		*out = new(int32)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchReleaseStatus.
@@ -162,8 +149,8 @@ func (in *BatchReleaseStatus) DeepCopy() *BatchReleaseStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CanaryStatus) DeepCopyInto(out *CanaryStatus) {
 	*out = *in
-	if in.LastReadyTime != nil {
-		in, out := &in.LastReadyTime, &out.LastReadyTime
+	if in.LastUpdateTime != nil {
+		in, out := &in.LastUpdateTime, &out.LastUpdateTime
 		*out = (*in).DeepCopy()
 	}
 }
@@ -181,16 +168,12 @@ func (in *CanaryStatus) DeepCopy() *CanaryStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CanaryStep) DeepCopyInto(out *CanaryStep) {
 	*out = *in
-	if in.SetWeight != nil {
-		in, out := &in.SetWeight, &out.SetWeight
-		*out = new(int32)
+	if in.CanaryReplicas != nil {
+		in, out := &in.CanaryReplicas, &out.CanaryReplicas
+		*out = new(intstr.IntOrString)
 		**out = **in
 	}
-	if in.Pause != nil {
-		in, out := &in.Pause, &out.Pause
-		*out = new(RolloutPause)
-		(*in).DeepCopyInto(*out)
-	}
+	in.Pause.DeepCopyInto(&out.Pause)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryStep.
@@ -230,14 +213,24 @@ func (in *CanaryStrategy) DeepCopy() *CanaryStrategy {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControllerRevisionRef) DeepCopyInto(out *ControllerRevisionRef) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerRevisionRef.
+func (in *ControllerRevisionRef) DeepCopy() *ControllerRevisionRef {
+	if in == nil {
+		return nil
+	}
+	out := new(ControllerRevisionRef)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NginxTrafficRouting) DeepCopyInto(out *NginxTrafficRouting) {
 	*out = *in
-	if in.Tickets != nil {
-		in, out := &in.Tickets, &out.Tickets
-		*out = new(Tickets)
-		(*in).DeepCopyInto(*out)
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NginxTrafficRouting.
@@ -253,6 +246,11 @@ func (in *NginxTrafficRouting) DeepCopy() *NginxTrafficRouting {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ObjectRef) DeepCopyInto(out *ObjectRef) {
 	*out = *in
+	if in.WorkloadRef != nil {
+		in, out := &in.WorkloadRef, &out.WorkloadRef
+		*out = new(WorkloadRef)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectRef.
@@ -306,6 +304,21 @@ func (in *ReleasePlan) DeepCopy() *ReleasePlan {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ReleaseStrategy) DeepCopyInto(out *ReleaseStrategy) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReleaseStrategy.
+func (in *ReleaseStrategy) DeepCopy() *ReleaseStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(ReleaseStrategy)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Rollout) DeepCopyInto(out *Rollout) {
 	*out = *in
@@ -405,7 +418,7 @@ func (in *RolloutPause) DeepCopy() *RolloutPause {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RolloutSpec) DeepCopyInto(out *RolloutSpec) {
 	*out = *in
-	out.TargetRef = in.TargetRef
+	in.ObjectRef.DeepCopyInto(&out.ObjectRef)
 	in.Strategy.DeepCopyInto(&out.Strategy)
 }
 
@@ -449,8 +462,8 @@ func (in *RolloutStatus) DeepCopy() *RolloutStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RolloutStrategy) DeepCopyInto(out *RolloutStrategy) {
 	*out = *in
-	if in.Canary != nil {
-		in, out := &in.Canary, &out.Canary
+	if in.CanaryPlan != nil {
+		in, out := &in.CanaryPlan, &out.CanaryPlan
 		*out = new(CanaryStrategy)
 		(*in).DeepCopyInto(*out)
 	}
@@ -466,47 +479,13 @@ func (in *RolloutStrategy) DeepCopy() *RolloutStrategy {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *Tickets) DeepCopyInto(out *Tickets) {
-	*out = *in
-	if in.Header != nil {
-		in, out := &in.Header, &out.Header
-		*out = make(map[string]string, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val
-		}
-	}
-	if in.Cookie != nil {
-		in, out := &in.Cookie, &out.Cookie
-		*out = make(map[string]string, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val
-		}
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Tickets.
-func (in *Tickets) DeepCopy() *Tickets {
-	if in == nil {
-		return nil
-	}
-	out := new(Tickets)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TrafficRouting) DeepCopyInto(out *TrafficRouting) {
 	*out = *in
 	if in.Nginx != nil {
 		in, out := &in.Nginx, &out.Nginx
 		*out = new(NginxTrafficRouting)
-		(*in).DeepCopyInto(*out)
-	}
-	if in.Alb != nil {
-		in, out := &in.Alb, &out.Alb
-		*out = new(AlbTrafficRouting)
-		(*in).DeepCopyInto(*out)
+		**out = **in
 	}
 }
 
@@ -519,3 +498,18 @@ func (in *TrafficRouting) DeepCopy() *TrafficRouting {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WorkloadRef) DeepCopyInto(out *WorkloadRef) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadRef.
+func (in *WorkloadRef) DeepCopy() *WorkloadRef {
+	if in == nil {
+		return nil
+	}
+	out := new(WorkloadRef)
+	in.DeepCopyInto(out)
+	return out
+}

config/crd/bases/_.yaml

@@ -0,0 +1,21 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.1
+  creationTimestamp: null
+spec:
+  group: ""
+  names:
+    kind: ""
+    plural: ""
+  scope: ""
+  versions: null
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: null
+  storedVersions: null

config/crd/bases/rollouts.kruise.io_batchreleases.yaml

@@ -6,9 +6,9 @@ metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.4.1
   creationTimestamp: null
-  name: batchreleases.apps.rollouts.io
+  name: batchreleases.rollouts.kruise.io
 spec:
-  group: apps.rollouts.io
+  group: rollouts.kruise.io
   names:
     kind: BatchRelease
     listKind: BatchReleaseList
@@ -49,73 +49,89 @@ spec:
           metadata:
             type: object
           spec:
-            description: BatchReleaseSpec defines how to describe a batch release
-              plan.
+            description: BatchReleaseSpec defines how to describe an update between
+              different compRevision
             properties:
               cancelled:
-                description: Cancelled is true indicates this batch release plan is
-                  cancelled. All resources about canary will be cleaned up.
+                description: Cancel is true indicates this batch release plan is cancelled.
                 type: boolean
               releasePlan:
-                description: ReleasePlan is the details on how to release the updated
-                  revision.
+                description: RolloutPlan is the details on how to rollout the resources
                 properties:
                   batchPartition:
                     description: All pods in the batches up to the batchPartition
-                      (included) will have the target updated specification while
-                      the rest still have the stable resource This is designed for
-                      the operators to manually release plan Default is nil, which
-                      means no partition.
+                      (included) will have the target resource specification while
+                      the rest still have the source resource This is designed for
+                      the operators to manually rollout Default is the the number
+                      of batches which will rollout all the batches
                     format: int32
                     type: integer
                   batches:
-                    description: Batches describe the plan for each batch in detail,
-                      including canary replicas and paused seconds.
+                    description: The exact distribution among batches. its size has
+                      to be exactly the same as the NumBatches (if set) The total
+                      number cannot exceed the targetSize or the size of the source
+                      resource We will IGNORE the last batch's replica field if it's
+                      a percentage since round errors can lead to inaccurate sum We
+                      highly recommend to leave the last batch's replica field empty
                     items:
-                      description: ReleaseBatch is used to describe how each release
-                        batch should be
+                      description: ReleaseBatch is used to describe how the each batch
+                        rollout should be
                       properties:
                         canaryReplicas:
                           anyOf:
                           - type: integer
                           - type: string
-                          description: 'Replicas is the number of expected canary
-                            pods in this batch it can be an absolute number (ex: 5)
-                            or a percentage of total pods. it is mutually exclusive
-                            with the PodList field'
+                          description: 'Replicas is the number of pods to upgrade
+                            in this batch it can be an absolute number (ex: 5) or
+                            a percentage of total pods we will ignore the percentage
+                            of the last batch to just fill the gap it is mutually
+                            exclusive with the PodList field'
                           x-kubernetes-int-or-string: true
                         pauseSeconds:
                           description: The wait time, in seconds, between instances
                             upgrades, default = 0
                           format: int64
                           type: integer
-                      required:
-                      - canaryReplicas
                       type: object
                     type: array
                   paused:
-                    description: Paused the release plan executor, default is false
+                    description: Paused the rollout, default is false
                     type: boolean
-                required:
-                - batches
+                type: object
+              strategy:
+                properties:
+                  cloneSetStrategy:
+                    type: string
+                  deploymentStrategy:
+                    type: string
                 type: object
               targetReference:
-                description: TargetRef contains the name of the workload that we need
-                  to upgrade to.
+                description: TargetRevisionName contains the name of the componentRevisionName
+                  that we need to upgrade to.
                 properties:
-                  apiVersion:
-                    description: API Version of the referent
-                    type: string
-                  kind:
-                    description: Kind of the referent
-                    type: string
-                  name:
-                    description: Name of the referent
+                  type:
+                    description: workloadRef, revisionRef default is workloadRef
                     type: string
+                  workloadRef:
+                    description: WorkloadRef contains enough information to let you
+                      identify a workload for Rollout Batch release of the bypass
+                    properties:
+                      apiVersion:
+                        description: API Version of the referent
+                        type: string
+                      kind:
+                        description: Kind of the referent
+                        type: string
+                      name:
+                        description: Name of the referent
+                        type: string
+                    required:
+                    - apiVersion
+                    - kind
+                    - name
+                    type: object
                 required:
-                - apiVersion
-                - kind
-                - name
+                - type
                 type: object
             required:
             - releasePlan
@@ -126,8 +142,12 @@ spec:
               plan
             properties:
               canaryStatus:
-                description: Canary describes the status of the canary
+                description: Canary describes the state of the canary rollout
                 properties:
+                  batchState:
+                    description: ReleasingBatchState indicates the state of the current
+                      batch.
+                    type: string
                   currentBatch:
                     description: The current batch the rollout is working on/blocked,
                       it starts from 0
@@ -137,9 +157,6 @@ spec:
                     description: LastBatchFinalizedTime is the timestamp of
                     format: date-time
                     type: string
-                  state:
-                    description: State indicates the state of the current batch.
-                    type: string
                   updatedReadyReplicas:
                     description: UpgradedReadyReplicas is the number of Pods upgraded
                       by the rollout controller that have a Ready Condition.
@@ -153,8 +170,15 @@ spec:
                 required:
                 - currentBatch
                 type: object
+              collisionCount:
+                description: Count of hash collisions for creating canary Deployment.
+                  The controller uses this field as a collision avoidance mechanism
+                  when it needs to create the name for the newest canary Deployment.
+                format: int32
+                type: integer
               conditions:
-                description: Conditions records some important message at each Phase.
+                description: Conditions represents the latest available observations
+                  of a CloneSet's current state.
                 items:
                   description: RolloutCondition describes the state of a rollout at
                     a certain point.

config/crd/bases/rollouts.kruise.io_rollouts.yaml

@@ -6,9 +6,9 @@ metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.4.1
   creationTimestamp: null
-  name: rollouts.apps.rollouts.io
+  name: rollouts.rollouts.kruise.io
 spec:
-  group: apps.rollouts.io
+  group: rollouts.kruise.io
   names:
     kind: Rollout
     listKind: RolloutList
@@ -36,28 +36,55 @@ spec:
           spec:
             description: RolloutSpec defines the desired state of Rollout
             properties:
+              objectRef:
+                properties:
+                  type:
+                    description: workloadRef, revisionRef default is workloadRef
+                    type: string
+                  workloadRef:
+                    description: WorkloadRef contains enough information to let you
+                      identify a workload for Rollout Batch release of the bypass
+                    properties:
+                      apiVersion:
+                        description: API Version of the referent
+                        type: string
+                      kind:
+                        description: Kind of the referent
+                        type: string
+                      name:
+                        description: Name of the referent
+                        type: string
+                    required:
+                    - apiVersion
+                    - kind
+                    - name
+                    type: object
+                required:
+                - type
+                type: object
               strategy:
                 description: The deployment strategy to use to replace existing pods
                   with new ones.
                 properties:
-                  canary:
-                    description: BlueGreen *BlueGreenStrategy `json:"blueGreen,omitempty"
-                      protobuf:"bytes,1,opt,name=blueGreen"`
+                  canaryPlan:
+                    description: CanaryStrategy defines parameters for a Replica Based
+                      Canary
                     properties:
-                      stableService:
-                        description: CanaryService holds the name of a service which
-                          selects pods with canary version and don't select any pods
-                          with stable version. CanaryService string `json:"canaryService,omitempty"`
-                          StableService holds the name of a service which selects
-                          pods with stable version and don't select any pods with
-                          canary version.
-                        type: string
                       steps:
                         description: Steps define the order of phases to execute the
                           canary deployment
                         items:
                           description: CanaryStep defines a step of a canary workload.
                           properties:
+                            canaryReplicas:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              description: 'Replicas is the number of expected canary
+                                pods in this batch it can be an absolute number (ex:
+                                5) or a percentage of total pods. it is mutually exclusive
+                                with the PodList field'
+                              x-kubernetes-int-or-string: true
                             pause:
                               description: Pause freezes the rollout by setting spec.Paused
                                 to true. A Rollout will resume when spec.Paused is
@@ -69,7 +96,7 @@ spec:
                                   format: int32
                                   type: integer
                               type: object
-                            setWeight:
+                            weight:
                               description: SetWeight sets what percentage of the canary
                                 pods should receive
                               format: int32
@@ -80,29 +107,6 @@ spec:
                         description: TrafficRouting hosts all the supported service
                           meshes supported to enable more fine-grained traffic routing
                         properties:
-                          alb:
-                            description: AlbTrafficRouting configuration for Nginx
-                              ingress controller to control traffic routing
-                            properties:
-                              ingress:
-                                description: Ingress refers to the name of an `Ingress`
-                                  resource in the same namespace as the `Rollout`
-                                type: string
-                              tickets:
-                                description: A/B Testing
-                                properties:
-                                  cookie:
-                                    additionalProperties:
-                                      type: string
-                                    type: object
-                                  header:
-                                    additionalProperties:
-                                      type: string
-                                    type: object
-                                type: object
-                            required:
-                            - ingress
-                            type: object
                           nginx:
                             description: Nginx holds Nginx Ingress specific configuration
                               to route traffic
@@ -111,51 +115,41 @@ spec:
                                 description: Ingress refers to the name of an `Ingress`
                                   resource in the same namespace as the `Rollout`
                                 type: string
-                              tickets:
-                                description: A/B Testing
-                                properties:
-                                  cookie:
-                                    additionalProperties:
-                                      type: string
-                                    type: object
-                                  header:
-                                    additionalProperties:
-                                      type: string
-                                    type: object
-                                type: object
                             required:
                             - ingress
                             type: object
-                          type:
+                          service:
+                            description: Service holds the name of a service which
+                              selects pods with stable version and don't select any
+                              pods with canary version.
                             type: string
+                          type:
+                            description: Nginx, Alb, Istio etc.
+                            type: string
+                        required:
+                        - type
                         type: object
                     type: object
-                type: object
-              targetRef:
-                description: TargetRef contains enough information to let you identify
-                  a workload for Rollout
-                properties:
-                  apiVersion:
-                    description: API Version of the referent
-                    type: string
-                  kind:
-                    description: Kind of the referent
-                    type: string
-                  name:
-                    description: Name of the referent
+                  paused:
+                    description: Paused indicates that the Rollout is paused. Default
+                      value is false
+                    type: boolean
+                  type:
+                    description: canaryPlan, BlueGreenPlan Default value is canaryPlan
                     type: string
                 required:
-                - apiVersion
-                - kind
-                - name
+                - type
                 type: object
             required:
+            - objectRef
             - strategy
-            - targetRef
             type: object
           status:
             description: RolloutStatus defines the observed state of Rollout
             properties:
+              canaryRevision:
+                description: CanaryRevision the hash of the canary pod template
+                type: string
               canaryStatus:
                 description: Canary describes the state of the canary rollout
                 properties:
@@ -247,9 +241,6 @@ spec:
                 description: StableRevision indicates the revision pods that has successfully
                   rolled out
                 type: string
-              updateRevision:
-                description: UpdateRevision the hash of the current pod template
-                type: string
             type: object
         type: object
     served: true

config/default/kustomization.yaml

@@ -1,12 +1,12 @@
 # Adds namespace to all resources.
-namespace: rollouts-system
+namespace: rollout-system
 
 # Value of this field is prepended to the
 # names of all resources, e.g. a deployment named
 # "wordpress" becomes "alices-wordpress".
 # Note that it should also match with the prefix (text before '-') of the namespace
 # field above.
-namePrefix: rollouts-
+namePrefix: rollout-
 
 # Labels to add to all resources and selectors.
 #commonLabels:
@@ -18,7 +18,7 @@ bases:
 - ../manager
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
-#- ../webhook
+- ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.

config/rbac/role.yaml

@@ -0,0 +1,222 @@
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: manager-role
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingwebhookconfigurations
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - apps
+  resources:
+  - replicasets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - apps
+  resources:
+  - replicasets/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - apps.kruise.io
+  resources:
+  - clonesets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - apps.kruise.io
+  resources:
+  - clonesets/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - pods/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - rollouts.kruise.io
+  resources:
+  - batchreleases
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - rollouts.kruise.io
+  resources:
+  - batchreleases/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - rollouts.kruise.io
+  resources:
+  - rollouts
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - rollouts.kruise.io
+  resources:
+  - rollouts/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - rollouts.kruise.io
+  resources:
+  - rollouts/status
+  verbs:
+  - get
+  - patch
+  - update

config/webhook/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+- manifests.yaml
+- service.yaml
+
+configurations:
+- kustomizeconfig.yaml

config/webhook/kustomizeconfig.yaml

@@ -0,0 +1,25 @@
+# the following config is for teaching kustomize where to look at when substituting vars.
+# It requires kustomize v2.1.0 or newer to work properly.
+nameReference:
+- kind: Service
+  version: v1
+  fieldSpecs:
+  - kind: MutatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
+  - kind: ValidatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
+
+namespace:
+- kind: MutatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true
+- kind: ValidatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true
+
+varReference:
+- path: metadata/annotations

config/webhook/manifests.yaml

@@ -0,0 +1,77 @@
+
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: mutating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-apps-kruise-io-v1alpha1-cloneset
+  failurePolicy: Fail
+  name: mcloneset.kb.io
+  rules:
+  - apiGroups:
+    - apps.kruise.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - UPDATE
+    resources:
+    - clonesets
+  sideEffects: None
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-apps-v1-deployment
+  failurePolicy: Fail
+  name: mdeployment.kb.io
+  rules:
+  - apiGroups:
+    - apps
+    apiVersions:
+    - v1
+    operations:
+    - UPDATE
+    resources:
+    - deployments
+  sideEffects: None
+
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: validating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-rollouts-kruise-io-rollout
+  failurePolicy: Fail
+  name: vrollout.kb.io
+  rules:
+  - apiGroups:
+    - rollouts.kruise.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - rollouts
+  sideEffects: None

config/webhook/service.yaml

@@ -0,0 +1,12 @@
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: webhook-service
+  namespace: system
+spec:
+  ports:
+    - port: 443
+      targetPort: 9876
+  selector:
+    control-plane: controller-manager

controllers/rollout/batchrelease/batchrelease.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package batchrelease
+
+type BatchController interface {
+	VerifyBatchInitial() (bool, string, error)
+
+	PromoteBatch(index int32) error
+
+	BatchReleaseState() (*BatchReleaseState, error)
+
+	Finalize() (bool, error)
+}

controllers/rollout/batchrelease/inner_controller.go

@@ -0,0 +1,204 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package batchrelease
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog/v2"
+	utilpointer "k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	// rollouts.kruise.io
+	BatchReleaseOwnerRefAnnotation = "rollouts.kruise.io/owner-ref"
+)
+
+type innerBatchController struct {
+	client.Client
+
+	rollout *appsv1alpha1.Rollout
+
+	batchName string
+}
+
+func NewInnerBatchController(c client.Client, rollout *appsv1alpha1.Rollout) BatchController {
+	r := &innerBatchController{
+		Client:    c,
+		rollout:   rollout,
+		batchName: rolloutBatchName(rollout),
+	}
+
+	return r
+}
+
+func (r *innerBatchController) VerifyBatchInitial() (bool, string, error) {
+	batch := &appsv1alpha1.BatchRelease{}
+	err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch)
+	if err != nil && !errors.IsNotFound(err) {
+		klog.Errorf("rollout(%s/%s) fetch BatchRelease(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.batchName, err.Error())
+		return false, "", err
+	} else if errors.IsNotFound(err) {
+		// create new BatchRelease Crd
+		br := createBatchRelease(r.rollout, r.batchName)
+		if err = r.Create(context.TODO(), br); err != nil && !errors.IsAlreadyExists(err) {
+			klog.Errorf("rollout(%s/%s) create BatchRelease(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.batchName, err.Error())
+			return false, "", err
+		}
+		klog.Infof("rollout(%s/%s) create BatchRelease(%s) success", r.rollout.Namespace, r.rollout.Name, r.batchName)
+		return false, "", nil
+	}
+
+	// verify batchRelease status
+	if batch.Status.ObservedGeneration == batch.Generation {
+		klog.Infof("rollout(%s/%s) batchRelease(%s) initialize done", r.rollout.Namespace, r.rollout.Name, r.batchName)
+		return true, "", nil
+	}
+	klog.Infof("rollout(%s/%s) batchRelease(%s) is initialing, and wait a moment ", r.rollout.Namespace, r.rollout.Name, r.batchName)
+	return false, "", nil
+}
+
+func (r *innerBatchController) BatchReleaseState() (*BatchReleaseState, error) {
+	batch := &appsv1alpha1.BatchRelease{}
+	err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch)
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) fetch batch(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.batchName, err.Error())
+		return nil, err
+	}
+
+	state := &BatchReleaseState{
+		UpdateRevision:       batch.Status.UpdateRevision,
+		StableRevision:       batch.Status.StableRevision,
+		CurrentBatch:         batch.Status.CanaryStatus.CurrentBatch,
+		UpdatedReplicas:      batch.Status.CanaryStatus.UpdatedReplicas,
+		UpdatedReadyReplicas: batch.Status.CanaryStatus.UpdatedReadyReplicas,
+		Paused:               batch.Spec.ReleasePlan.Paused,
+	}
+	if batch.Status.CanaryStatus.ReleasingBatchState == appsv1alpha1.ReadyBatchState {
+		state.State = BatchReadyState
+	} else {
+		state.State = BatchInRollingState
+	}
+	return state, nil
+}
+
+func (r *innerBatchController) PromoteBatch(index int32) error {
+	batch := &appsv1alpha1.BatchRelease{}
+	if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		if err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch); err != nil {
+			klog.Errorf("error getting updated rollout(%s/%s) from client", batch.Namespace, batch.Name)
+			return err
+		}
+		if !batch.Spec.ReleasePlan.Paused && *batch.Spec.ReleasePlan.BatchPartition == index {
+			return nil
+		}
+		batch.Spec.ReleasePlan.BatchPartition = utilpointer.Int32Ptr(index)
+		batch.Spec.ReleasePlan.Paused = false
+		if err := r.Client.Update(context.TODO(), batch); err != nil {
+			return err
+		}
+		klog.Infof("rollout(%s/%s) promote batchRelease(%s) BatchPartition(%d) success", r.rollout.Namespace, r.rollout.Name, r.batchName, index)
+		return nil
+	}); err != nil {
+		klog.Errorf("rollout(%s/%s) promote batchRelease(%s) BatchPartition(%d) failed: %s", r.rollout.Namespace, r.rollout.Name, r.batchName, index, err.Error())
+		return err
+	}
+	return nil
+}
+
+func (r *innerBatchController) Finalize() (bool, error) {
+	batch := &appsv1alpha1.BatchRelease{}
+	err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: r.batchName}, batch)
+	if err != nil && errors.IsNotFound(err) {
+		klog.Infof("rollout(%s/%s) delete batch(%s) success", r.rollout.Namespace, r.rollout.Name, r.batchName)
+		return true, nil
+	} else if err != nil {
+		klog.Errorf("rollout(%s/%s) fetch batch failed: %s", r.rollout.Namespace, r.rollout.Name, r.batchName)
+		return false, err
+	}
+
+	if !batch.DeletionTimestamp.IsZero() {
+		klog.Infof("rollout(%s/%s) batch(%s) is terminating, and wait a moment", r.rollout.Namespace, r.rollout.Name, r.batchName)
+		return false, nil
+	}
+
+	//delete batchRelease
+	err = r.Delete(context.TODO(), batch)
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) delete batch(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.batchName, err.Error())
+		return false, err
+	}
+	klog.Infof("rollout(%s/%s) delete batch(%s), and wait a moment", r.rollout.Namespace, r.rollout.Name, r.batchName)
+	return false, nil
+}
+
+func createBatchRelease(rollout *appsv1alpha1.Rollout, batchName string) *appsv1alpha1.BatchRelease {
+	var batches []appsv1alpha1.ReleaseBatch
+	var lastBatch appsv1alpha1.ReleaseBatch
+	for _, step := range rollout.Spec.Strategy.CanaryPlan.Steps {
+		batches = append(batches, appsv1alpha1.ReleaseBatch{CanaryReplicas: intstr.FromString(strconv.Itoa(int(step.Weight)) + "%")})
+		lastBatch = appsv1alpha1.ReleaseBatch{CanaryReplicas: intstr.FromString(strconv.Itoa(int(step.Weight)) + "%")}
+	}
+	batches = append(batches, lastBatch)
+
+	br := &appsv1alpha1.BatchRelease{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: rollout.Namespace,
+			Name:      batchName,
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(rollout, schema.GroupVersionKind{
+					Group:   appsv1alpha1.SchemeGroupVersion.Group,
+					Version: appsv1alpha1.SchemeGroupVersion.Version,
+					Kind:    "Rollout",
+				}),
+			},
+			Annotations: map[string]string{
+				BatchReleaseOwnerRefAnnotation: rollout.Name,
+			},
+		},
+		Spec: appsv1alpha1.BatchReleaseSpec{
+			TargetRef: appsv1alpha1.ObjectRef{
+				Type: appsv1alpha1.WorkloadRefType,
+				WorkloadRef: &appsv1alpha1.WorkloadRef{
+					APIVersion: rollout.Spec.ObjectRef.WorkloadRef.APIVersion,
+					Kind:       rollout.Spec.ObjectRef.WorkloadRef.Kind,
+					Name:       rollout.Spec.ObjectRef.WorkloadRef.Name,
+				},
+			},
+			ReleasePlan: appsv1alpha1.ReleasePlan{
+				Batches:        batches,
+				BatchPartition: utilpointer.Int32Ptr(0),
+				Paused:         true,
+			},
+		},
+	}
+	return br
+}
+
+// {workload.name}-batch
+func rolloutBatchName(rollout *appsv1alpha1.Rollout) string {
+	return fmt.Sprintf("%s-batch", rollout.Spec.ObjectRef.WorkloadRef.Name)
+}

controllers/rollout/batchrelease/types.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package batchrelease
+
+type BatchReleaseState struct {
+	// UpdateRevision the hash of the current pod template
+	UpdateRevision string
+	// StableRevision indicates the revision pods that has successfully rolled out
+	StableRevision string
+	// current batch step
+	CurrentBatch         int32
+	UpdatedReplicas      int32
+	UpdatedReadyReplicas int32
+	Paused               bool
+	State                BatchRollingState
+}
+
+type BatchRollingState string
+
+const (
+	BatchInRollingState BatchRollingState = "batchInRolling"
+	// BatchReadyState indicates that all the pods in the are upgraded and its state is ready
+	BatchReadyState BatchRollingState = "batchReady"
+)

controllers/rollout/canary.go

@@ -0,0 +1,299 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rollout
+
+import (
+	"context"
+	"encoding/json"
+	kruiseappsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1"
+	apps "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"strconv"
+	"time"
+
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	"github.com/openkruise/rollouts/controllers/rollout/batchrelease"
+	"github.com/openkruise/rollouts/pkg/util"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/klog/v2"
+)
+
+func (r *rolloutContext) runCanary() error {
+	canaryStatus := r.newStatus.CanaryStatus
+	// In case of continuous publishing(v1 -> v2 -> v3), then restart publishing
+	if r.newStatus.CanaryRevision != r.newStatus.CanaryStatus.CanaryRevision {
+		canaryStatus.CurrentStepState = appsv1alpha1.CanaryStepStateUpgrade
+		canaryStatus.CanaryRevision = r.workload.CanaryRevision
+		canaryStatus.CurrentStepIndex = 0
+		canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()}
+		canaryStatus.CanaryReplicas = 0
+		canaryStatus.CanaryReadyReplicas = 0
+		klog.Infof("rollout(%s/%s) workload continuous publishing canaryRevision from(%s) -> to(%s), then restart publishing",
+			r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CanaryRevision, r.newStatus.CanaryRevision)
+		return nil
+	}
+
+	// update canary status
+	r.newStatus.CanaryStatus.CanaryReplicas = r.workload.CanaryReplicas
+	r.newStatus.CanaryStatus.CanaryReadyReplicas = r.workload.CanaryReadyReplicas
+	switch canaryStatus.CurrentStepState {
+	case appsv1alpha1.CanaryStepStateUpgrade:
+		klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, appsv1alpha1.CanaryStepStateUpgrade)
+		done, err := r.doCanaryUpgrade()
+		if err != nil {
+			return err
+		} else if done {
+			canaryStatus.CurrentStepState = appsv1alpha1.CanaryStepStateTrafficRouting
+			canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()}
+			klog.Infof("rollout(%s/%s) step(index:%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name,
+				canaryStatus.CurrentStepIndex, appsv1alpha1.CanaryStepStateUpgrade, canaryStatus.CurrentStepState)
+		}
+	case appsv1alpha1.CanaryStepStateTrafficRouting:
+		klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, appsv1alpha1.CanaryStepStateTrafficRouting)
+		done, err := r.doCanaryTrafficRouting()
+		if err != nil {
+			return err
+		} else if done {
+			canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()}
+			canaryStatus.CurrentStepState = appsv1alpha1.CanaryStepStateMetricsAnalysis
+			klog.Infof("rollout(%s/%s) step(index:%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name,
+				canaryStatus.CurrentStepIndex, appsv1alpha1.CanaryStepStateTrafficRouting, canaryStatus.CurrentStepState)
+		}
+		expectedTime := time.Now().Add(5 * time.Second)
+		r.recheckTime = &expectedTime
+
+	case appsv1alpha1.CanaryStepStateMetricsAnalysis:
+		klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, appsv1alpha1.CanaryStepStateMetricsAnalysis)
+		done, err := r.doCanaryMetricsAnalysis()
+		if err != nil {
+			return err
+		} else if done {
+			canaryStatus.CurrentStepState = appsv1alpha1.CanaryStepStatePaused
+			klog.Infof("rollout(%s/%s) step(index:%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name,
+				canaryStatus.CurrentStepIndex, appsv1alpha1.CanaryStepStateMetricsAnalysis, canaryStatus.CurrentStepState)
+		}
+
+	case appsv1alpha1.CanaryStepStatePaused:
+		klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, appsv1alpha1.CanaryStepStatePaused)
+		done, err := r.doCanaryPaused()
+		if err != nil {
+			return err
+		} else if done {
+			canaryStatus.LastUpdateTime = &metav1.Time{Time: time.Now()}
+			canaryStatus.CurrentStepState = appsv1alpha1.CanaryStepStateCompleted
+			klog.Infof("rollout(%s/%s) step(index:%d) state from(%s) -> to(%s)", r.rollout.Namespace, r.rollout.Name,
+				canaryStatus.CurrentStepIndex, appsv1alpha1.CanaryStepStatePaused, canaryStatus.CurrentStepState)
+		}
+
+	case appsv1alpha1.CanaryStepStateCompleted:
+		klog.Infof("rollout(%s/%s) run canary strategy, and state(%s)", r.rollout.Namespace, r.rollout.Name, appsv1alpha1.CanaryStepStateCompleted)
+		if len(r.rollout.Spec.Strategy.CanaryPlan.Steps) > int(canaryStatus.CurrentStepIndex+1) {
+			canaryStatus.CurrentStepIndex++
+			canaryStatus.CurrentStepState = appsv1alpha1.CanaryStepStateUpgrade
+			klog.Infof("rollout(%s/%s) canary step from(%d) -> to(%d)", r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex-1, canaryStatus.CurrentStepIndex)
+		} else {
+			klog.Infof("rollout(%s/%s) canary run all steps, and completed", r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex-1, canaryStatus.CurrentStepIndex)
+		}
+	}
+
+	return nil
+}
+
+func (r *rolloutContext) doCanaryUpgrade() (bool, error) {
+	// only traffic routing
+	/*if len(r.rollout.Spec.Strategy.CanaryPlan.Steps) == 0 {
+		if r.workload.CanaryReadyReplicas > 0 {
+			klog.Infof("rollout(%s/%s) workload(%s) canaryAvailable(%d), and go to the next stage",
+				r.rollout.Namespace, r.rollout.Name, r.workload.Name, r.workload.CanaryReadyReplicas)
+			return true, nil
+		}
+		klog.Infof("rollout(%s/%s) workload(%s) canaryAvailable(%d), and wait a moment",
+			r.rollout.Namespace, r.rollout.Name, r.workload.Name, r.workload.CanaryReadyReplicas)
+		return false, nil
+	}*/
+
+	// canary release
+	batchState, err := r.batchControl.BatchReleaseState()
+	if err != nil {
+		return false, err
+	}
+	canaryStatus := r.newStatus.CanaryStatus
+	// promote workload next batch release
+	if batchState.Paused || batchState.CurrentBatch < canaryStatus.CurrentStepIndex {
+		klog.Infof("rollout(%s/%s) will promote batch from(%d) -> to(%d)", r.rollout.Namespace, r.rollout.Name, batchState.CurrentBatch, canaryStatus.CurrentStepIndex)
+		return false, r.batchControl.PromoteBatch(canaryStatus.CurrentStepIndex)
+	}
+
+	// check whether batchRelease is ready
+	if batchState.State == batchrelease.BatchInRollingState {
+		klog.Infof("rollout(%s/%s) workload(%s) batch(%d) availableReplicas(%d) state(%s), and wait a moment",
+			r.rollout.Namespace, r.rollout.Name, r.workload.Name, canaryStatus.CurrentStepIndex, batchState.UpdatedReadyReplicas, batchState.State)
+		return false, nil
+	}
+	klog.Infof("rollout(%s/%s) workload(%s) batch(%d) availableReplicas(%d) state(%s), and continue",
+		r.rollout.Namespace, r.rollout.Name, r.workload.Name, canaryStatus.CurrentStepIndex, batchState.UpdatedReadyReplicas, batchState.State)
+	return true, nil
+}
+
+func (r *rolloutContext) doCanaryMetricsAnalysis() (bool, error) {
+	// todo
+	return true, nil
+}
+
+func (r *rolloutContext) doCanaryPaused() (bool, error) {
+	// No step set, need manual confirmation
+	if len(r.rollout.Spec.Strategy.CanaryPlan.Steps) == 0 {
+		klog.Infof("rollout(%s/%s) don't contains steps, and need manual confirmation", r.rollout.Namespace, r.rollout.Name)
+		return false, nil
+	}
+
+	currentStep := r.rollout.Spec.Strategy.CanaryPlan.Steps[r.newStatus.CanaryStatus.CurrentStepIndex]
+	// need manual confirmation
+	if currentStep.Pause.Duration == nil {
+		klog.Infof("rollout(%s/%s) don't set pause duration, and need manual confirmation", r.rollout.Namespace, r.rollout.Name)
+		return false, nil
+	}
+
+	// wait duration time, then go to next step
+	duration := time.Second * time.Duration(*currentStep.Pause.Duration)
+	expectedTime := r.newStatus.CanaryStatus.LastUpdateTime.Add(duration)
+	if expectedTime.Before(time.Now()) {
+		klog.Infof("rollout(%s/%s) canary step(%d) paused duration(%d seconds), and go to the next step",
+			r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex, *currentStep.Pause.Duration)
+		return true, nil
+	} else {
+		if r.recheckTime == nil || expectedTime.Before(*r.recheckTime) {
+			r.recheckTime = &expectedTime
+		}
+	}
+	return false, nil
+}
+
+// cleanup after rollout is completed or finished
+func (r *rolloutContext) doCanaryFinalising(isPromote bool) (bool, error) {
+	// when CanaryStatus is nil, which means canary action hasn't started yet, don't need doing cleanup
+	if r.newStatus.CanaryStatus == nil {
+		return true, nil
+	}
+
+	// 1. remove stable service podRevision selector
+	if r.rollout.Spec.Strategy.CanaryPlan.TrafficRouting != nil {
+		done, err := r.restoreStableService()
+		if err != nil || !done {
+			return done, err
+		}
+	}
+
+	// 2. mark rollout process complete, allow workload paused=false in webhook
+	if err := r.updateRolloutStateInWorkload(util.RolloutState{RolloutName: r.rollout.Name, RolloutDone: true}, false); err != nil {
+		return false, err
+	}
+
+	// 3. after the normal rollout is completed, first need to upgrade stable deployment to new revision
+	if isPromote {
+		batchState, err := r.batchControl.BatchReleaseState()
+		if err != nil && !errors.IsNotFound(err) {
+			return false, err
+		} else if batchState != nil && r.workload != nil {
+			if batchState.CurrentBatch < int32(len(r.rollout.Spec.Strategy.CanaryPlan.Steps)) {
+				klog.Infof("rollout(%s/%s) will promote batch from(%d) -> to(%d)", r.rollout.Namespace, r.rollout.Name,
+					batchState.CurrentBatch, len(r.rollout.Spec.Strategy.CanaryPlan.Steps))
+				return false, r.batchControl.PromoteBatch(int32(len(r.rollout.Spec.Strategy.CanaryPlan.Steps)))
+			}
+			if batchState.State == batchrelease.BatchInRollingState {
+				klog.Infof("rollout(%s/%s) batch(%d) availableReplicas(%d) state(%s), and wait a moment",
+					r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex, batchState.UpdatedReadyReplicas, batchState.State)
+				return false, nil
+			}
+		}
+	}
+
+	// 3. route all traffic to stable service
+	if r.rollout.Spec.Strategy.CanaryPlan.TrafficRouting != nil {
+		done, err := r.doFinalisingTrafficRouting()
+		if err != nil || !done {
+			return done, err
+		}
+	}
+
+	// 4. delete batchRelease CRD
+	done, err := r.batchControl.Finalize()
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) DoFinalising batchRelease failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error())
+		return false, err
+	} else if !done {
+		return false, nil
+	}
+
+	// delete rolloutState in workload
+	if err := r.updateRolloutStateInWorkload(util.RolloutState{}, true); err != nil {
+		return false, err
+	}
+	klog.Infof("rollout(%s/%s) DoFinalising batchRelease success", r.rollout.Namespace, r.rollout.Name)
+	return true, nil
+}
+
+func (r *rolloutContext) getDesiredAvailableForCanary() (int, error) {
+	currentStep := r.rollout.Spec.Strategy.CanaryPlan.Steps[r.newStatus.CanaryStatus.CurrentStepIndex]
+	weight := intstr.FromString(strconv.Itoa(int(currentStep.Weight)) + "%")
+	return intstr.GetScaledValueFromIntOrPercent(&weight, int(r.workload.WorkloadReplicas), true)
+}
+
+func (r *rolloutContext) podRevisionLabelKey() string {
+	if r.rollout.Spec.ObjectRef.WorkloadRef.Kind == util.ControllerKruiseKindCS.Kind {
+		return util.CloneSetPodRevisionLabelKey
+	}
+	return util.RsPodRevisionLabelKey
+}
+
+func (r *rolloutContext) updateRolloutStateInWorkload(state util.RolloutState, isDelete bool) error {
+	if r.workload == nil {
+		return nil
+	}
+	var obj client.Object
+	// cloneSet
+	if r.workload.Kind == util.ControllerKruiseKindCS.Kind {
+		obj = &kruiseappsv1alpha1.CloneSet{}
+		// deployment
+	} else {
+		obj = &apps.Deployment{}
+	}
+	by, _ := json.Marshal(state)
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		if err := r.Get(context.TODO(), types.NamespacedName{Name: r.workload.Name, Namespace: r.workload.Namespace}, obj); err != nil {
+			klog.Errorf("getting updated workload(%s.%s) failed: %s", r.workload.Namespace, r.workload.Name, err.Error())
+			return err
+		}
+		annotations := obj.GetAnnotations()
+		if isDelete {
+			delete(annotations, util.InRolloutProgressingAnnotation)
+		} else {
+			annotations[util.InRolloutProgressingAnnotation] = string(by)
+		}
+		return r.Update(context.TODO(), obj)
+	})
+	if err != nil {
+		klog.Errorf("update rollout(%s/%s) workload(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.workload.Name, err.Error())
+		return err
+	}
+	klog.Infof("update rollout(%s/%s) workload(%s) state(%s) delete(%t) success", r.rollout.Namespace, r.rollout.Name, r.workload.Name, string(by), isDelete)
+	return nil
+}

controllers/rollout/context.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rollout
+
+import (
+	"github.com/openkruise/rollouts/pkg/util"
+	"k8s.io/klog/v2"
+	"time"
+
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	"github.com/openkruise/rollouts/controllers/rollout/batchrelease"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type rolloutContext struct {
+	client.Client
+
+	rollout *appsv1alpha1.Rollout
+
+	newStatus *appsv1alpha1.RolloutStatus
+
+	stableRevision string
+
+	canaryRevision string
+
+	stableService *corev1.Service
+
+	canaryService *corev1.Service
+
+	workload *util.Workload
+
+	batchControl batchrelease.BatchController
+
+	recheckTime *time.Time
+}
+
+func (r *rolloutContext) reconcile() error {
+	// canary strategy
+	if r.rollout.Spec.Strategy.CanaryPlan != nil {
+		klog.Infof("rollout(%s/%s) run CanaryPlan action...", r.rollout.Namespace, r.rollout.Name)
+		return r.runCanary()
+	}
+	return nil
+}
+
+func (r *rolloutContext) finalising(isPromote bool) (bool, error) {
+	// canary strategy
+	if r.rollout.Spec.Strategy.CanaryPlan != nil {
+		done, err := r.doCanaryFinalising(isPromote)
+		if err == nil && !done {
+			// The finalizer is not finished, wait one second
+			expectedTime := time.Now().Add(5 * time.Second)
+			r.recheckTime = &expectedTime
+		}
+		return done, err
+	}
+	return false, nil
+}

controllers/rollout/finalizer.go

@@ -0,0 +1,141 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rollout
+
+import (
+	"context"
+	"github.com/openkruise/rollouts/pkg/util"
+	"time"
+
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	"github.com/openkruise/rollouts/controllers/rollout/batchrelease"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func (r *RolloutReconciler) reconcileRolloutTerminating(rollout *appsv1alpha1.Rollout) (*time.Time, error) {
+	cond := util.GetRolloutCondition(rollout.Status, appsv1alpha1.RolloutConditionTerminating)
+	klog.Infof("reconcile rollout(%s/%s) Terminating action", rollout.Namespace, rollout.Name)
+	if cond.Reason == appsv1alpha1.TerminatingReasonCompleted {
+		return nil, nil
+	}
+
+	newStatus := rollout.Status.DeepCopy()
+	done, recheckTime, err := r.doFinalising(rollout, newStatus, false)
+	if err != nil {
+		return nil, err
+	} else if done {
+		klog.Infof("rollout(%s/%s) is terminating, and state from(%s) -> to(%s)", rollout.Namespace, rollout.Name, cond.Reason, appsv1alpha1.TerminatingReasonCompleted)
+		cond.Reason = appsv1alpha1.TerminatingReasonCompleted
+		cond.Status = corev1.ConditionTrue
+		util.SetRolloutCondition(newStatus, *cond)
+	}
+	err = r.updateRolloutStatus(rollout, *newStatus)
+	if err != nil {
+		klog.Errorf("update rollout(%s/%s) status failed: %s", rollout.Namespace, rollout.Name, err.Error())
+		return nil, err
+	}
+	return recheckTime, nil
+}
+
+func (r *RolloutReconciler) reconcileRolloutRollback(rollout *appsv1alpha1.Rollout) (*time.Time, error) {
+	cond := util.GetRolloutCondition(rollout.Status, appsv1alpha1.RolloutConditionRollback)
+	klog.Infof("reconcile rollout(%s/%s) Rollback action", rollout.Namespace, rollout.Name)
+	if cond.Reason == appsv1alpha1.RollbackReasonCompleted {
+		return nil, nil
+	}
+
+	newStatus := rollout.Status.DeepCopy()
+	done, recheckTime, err := r.doFinalising(rollout, newStatus, false)
+	if err != nil {
+		return nil, err
+	} else if done {
+		klog.Infof("rollout(%s/%s) is terminating, and state from(%s) -> to(%s)", rollout.Namespace, rollout.Name, cond.Reason, appsv1alpha1.TerminatingReasonCompleted)
+		cond.Reason = appsv1alpha1.TerminatingReasonCompleted
+		cond.Status = corev1.ConditionTrue
+		util.SetRolloutCondition(newStatus, *cond)
+	}
+	err = r.updateRolloutStatus(rollout, *newStatus)
+	if err != nil {
+		klog.Errorf("update rollout(%s/%s) status failed: %s", rollout.Namespace, rollout.Name, err.Error())
+		return nil, err
+	}
+	return recheckTime, nil
+}
+
+func (r *RolloutReconciler) doFinalising(rollout *appsv1alpha1.Rollout, newStatus *appsv1alpha1.RolloutStatus, isPromote bool) (bool, *time.Time, error) {
+	// fetch target workload
+	workload, err := r.Finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef)
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) GetWorkloadForRef failed: %s", rollout.Namespace, rollout.Name, err.Error())
+		return false, nil, err
+	}
+
+	rolloutCon := &rolloutContext{
+		Client:         r.Client,
+		rollout:        rollout,
+		newStatus:      newStatus,
+		stableRevision: newStatus.StableRevision,
+		canaryRevision: newStatus.CanaryRevision,
+		batchControl:   batchrelease.NewInnerBatchController(r.Client, rollout),
+		workload:       workload,
+	}
+	done, err := rolloutCon.finalising(isPromote)
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) Progressing failed: %s", rollout.Namespace, rollout.Name, err.Error())
+		return false, nil, err
+	} else if !done {
+		klog.Infof("rollout(%s/%s) finalizer is not finished, and time(%s) retry reconcile", rollout.Namespace, rollout.Name, rolloutCon.recheckTime.String())
+		return false, rolloutCon.recheckTime, nil
+	}
+	//newStatus.CanaryStatus = nil
+	klog.Infof("run rollout(%s/%s) Progressing Finalising done", rollout.Namespace, rollout.Name)
+	return true, nil, nil
+}
+
+// handle adding and handle finalizer logic, it turns if we should continue to reconcile
+func (r *RolloutReconciler) handleFinalizer(rollout *appsv1alpha1.Rollout) (bool, error) {
+	if !rollout.DeletionTimestamp.IsZero() {
+		cond := util.GetRolloutCondition(rollout.Status, appsv1alpha1.RolloutConditionTerminating)
+		if cond != nil && cond.Reason == appsv1alpha1.TerminatingReasonCompleted {
+			// Completed
+			if controllerutil.ContainsFinalizer(rollout, util.KruiseRolloutFinalizer) {
+				controllerutil.RemoveFinalizer(rollout, util.KruiseRolloutFinalizer)
+				err := r.Update(context.TODO(), rollout)
+				if err != nil {
+					klog.Errorf("remove rollout(%s/%s) finalizer failed: %s", rollout.Namespace, rollout.Name, err.Error())
+					return false, err
+				}
+				klog.Infof("remove rollout(%s/%s) finalizer success", rollout.Namespace, rollout.Name)
+			}
+			return true, nil
+		}
+		return false, nil
+	}
+
+	if !controllerutil.ContainsFinalizer(rollout, util.KruiseRolloutFinalizer) {
+		controllerutil.AddFinalizer(rollout, util.KruiseRolloutFinalizer)
+		err := r.Update(context.TODO(), rollout)
+		if err != nil {
+			klog.Errorf("register rollout(%s/%s) finalizer failed: %s", rollout.Namespace, rollout.Name, err.Error())
+			return false, err
+		}
+		klog.Infof("register rollout(%s/%s) finalizer success", rollout.Namespace, rollout.Name)
+	}
+	return false, nil
+}

controllers/rollout/progressing.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rollout
+
+import (
+	"context"
+	"fmt"
+	"github.com/openkruise/rollouts/pkg/util"
+	"time"
+
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	"github.com/openkruise/rollouts/controllers/rollout/batchrelease"
+	corev1 "k8s.io/api/core/v1"
+	netv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog/v2"
+)
+
+// parameter1 retryReconcile, parameter2 error
+func (r *RolloutReconciler) reconcileRolloutProgressing(rollout *appsv1alpha1.Rollout) (*time.Time, error) {
+	cond := util.GetRolloutCondition(rollout.Status, appsv1alpha1.RolloutConditionProgressing)
+	klog.Infof("reconcile rollout(%s/%s) progressing action", rollout.Namespace, rollout.Name)
+
+	var err error
+	var recheckTime *time.Time
+	newStatus := rollout.Status.DeepCopy()
+	switch cond.Reason {
+	case appsv1alpha1.ProgressingReasonInitializing:
+		klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason)
+		done, msg, err := r.doProgressingInitializing(rollout)
+		if err != nil {
+			klog.Errorf("rollout(%s/%s) doProgressingInitializing error(%s)", rollout.Namespace, rollout.Name, err.Error())
+			return nil, err
+		} else if done {
+			progressingStateTransition(newStatus, corev1.ConditionFalse, appsv1alpha1.ProgressingReasonInRolling, "rollout is InRolling")
+		} else {
+			// Incomplete, recheck
+			expectedTime := time.Now().Add(5 * time.Second)
+			recheckTime = &expectedTime
+			progressingStateTransition(newStatus, corev1.ConditionFalse, appsv1alpha1.ProgressingReasonInitializing, msg)
+			klog.Infof("rollout(%s/%s) doProgressingInitializing is incomplete, and recheck(%s)", rollout.Namespace, rollout.Name, expectedTime.String())
+		}
+
+	case appsv1alpha1.ProgressingReasonInRolling:
+		// pause rollout
+		if rollout.Spec.Strategy.Paused {
+			klog.Infof("rollout(%s/%s) is Progressing, but paused", rollout.Namespace, rollout.Name)
+			progressingStateTransition(newStatus, corev1.ConditionFalse, appsv1alpha1.ProgressingReasonPaused, "rollout is paused")
+		} else {
+			klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason)
+			//check if progressing is done
+			if len(rollout.Spec.Strategy.CanaryPlan.Steps) == int(newStatus.CanaryStatus.CurrentStepIndex+1) &&
+				newStatus.CanaryStatus.CurrentStepState == appsv1alpha1.CanaryStepStateCompleted {
+				klog.Infof("rollout(%s/%s) progressing rolling done", rollout.Namespace, rollout.Name)
+				progressingStateTransition(newStatus, corev1.ConditionTrue, appsv1alpha1.ProgressingReasonFinalising, "")
+			} else { // rollout is in rolling
+				recheckTime, err = r.doProgressingInRolling(rollout, newStatus)
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+
+	case appsv1alpha1.ProgressingReasonFinalising:
+		klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason)
+		var done bool
+		done, recheckTime, err = r.doFinalising(rollout, newStatus, true)
+		if err != nil {
+			return nil, err
+			// finalizer is finished
+		} else if done {
+			progressingStateTransition(newStatus, corev1.ConditionTrue, appsv1alpha1.ProgressingReasonSucceeded, "")
+		}
+
+	case appsv1alpha1.ProgressingReasonPaused:
+		if !rollout.Spec.Strategy.Paused {
+			klog.Infof("rollout(%s/%s) is Progressing, but paused", rollout.Namespace, rollout.Name)
+			progressingStateTransition(newStatus, corev1.ConditionFalse, appsv1alpha1.ProgressingReasonInRolling, "rollout is InRolling")
+		}
+
+	case appsv1alpha1.ProgressingReasonSucceeded, appsv1alpha1.ProgressingReasonCanceled:
+		klog.Infof("rollout(%s/%s) is Progressing, and in reason(%s)", rollout.Namespace, rollout.Name, cond.Reason)
+	}
+
+	err = r.updateRolloutStatus(rollout, *newStatus)
+	if err != nil {
+		klog.Errorf("update rollout(%s/%s) status failed: %s", rollout.Namespace, rollout.Name, err.Error())
+		return nil, err
+	}
+	return recheckTime, nil
+}
+
+func progressingStateTransition(status *appsv1alpha1.RolloutStatus, condStatus corev1.ConditionStatus, reason, message string) {
+	cond := util.NewRolloutCondition(appsv1alpha1.RolloutConditionProgressing, condStatus, reason, message)
+	util.SetRolloutCondition(status, cond)
+}
+
+func (r *RolloutReconciler) doProgressingInitializing(rollout *appsv1alpha1.Rollout) (bool, string, error) {
+	if rollout.Spec.Strategy.Type == "" || rollout.Spec.Strategy.Type == appsv1alpha1.RolloutStrategyCanary {
+		if ok, msg, err := r.verifyCanaryStrategy(rollout); !ok {
+			return ok, msg, err
+		}
+		klog.Infof("verify rollout(%s/%s) CanaryStrategy success", rollout.Namespace, rollout.Name)
+	}
+
+	return true, "", nil
+}
+
+func (r *RolloutReconciler) doProgressingInRolling(rollout *appsv1alpha1.Rollout, newStatus *appsv1alpha1.RolloutStatus) (*time.Time, error) {
+	// fetch target workload
+	workload, err := r.Finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef)
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) GetWorkloadForRef failed: %s", rollout.Namespace, rollout.Name, err.Error())
+		return nil, err
+	} else if workload == nil {
+		expectedTime := time.Now().Add(5 * time.Second)
+		klog.Warningf("rollout(%s/%s) Fetch workload Not Found, and recheck(%s)", rollout.Namespace, rollout.Name, expectedTime.String())
+		return &expectedTime, nil
+	}
+
+	rolloutCon := &rolloutContext{
+		Client:         r.Client,
+		rollout:        rollout,
+		newStatus:      newStatus,
+		stableRevision: newStatus.StableRevision,
+		canaryRevision: newStatus.CanaryRevision,
+		workload:       workload,
+		batchControl:   batchrelease.NewInnerBatchController(r.Client, rollout),
+	}
+	err = rolloutCon.reconcile()
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) Progressing failed: %s", rollout.Namespace, rollout.Name, err.Error())
+		return nil, err
+	}
+	return rolloutCon.recheckTime, nil
+}
+
+func (r *RolloutReconciler) verifyCanaryStrategy(rollout *appsv1alpha1.Rollout) (bool, string, error) {
+	canary := rollout.Spec.Strategy.CanaryPlan
+	// Traffic routing
+	if canary.TrafficRouting != nil {
+		if ok, msg, err := r.verifyTrafficRouting(rollout.Namespace, canary.TrafficRouting); !ok {
+			return ok, msg, err
+		}
+	}
+
+	// canary steps
+	if len(canary.Steps) != 0 {
+		// create batch release crd
+		batchControl := batchrelease.NewInnerBatchController(r.Client, rollout)
+		return batchControl.VerifyBatchInitial()
+	}
+
+	return true, "", nil
+}
+
+func (r *RolloutReconciler) verifyTrafficRouting(ns string, tr *appsv1alpha1.TrafficRouting) (bool, string, error) {
+	// check service
+	service := &corev1.Service{}
+	err := r.Get(context.TODO(), types.NamespacedName{Namespace: ns, Name: tr.Service}, service)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return false, fmt.Sprintf("Service(%s/%s) is Not Found", ns, tr.Service), nil
+		}
+		return false, "", err
+	}
+
+	// check ingress
+	var ingressName string
+	switch tr.Type {
+	case appsv1alpha1.TrafficRoutingNginx:
+		nginx := tr.Nginx
+		ingressName = nginx.Ingress
+	}
+	ingress := &netv1.Ingress{}
+	err = r.Get(context.TODO(), types.NamespacedName{Namespace: ns, Name: ingressName}, ingress)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return false, fmt.Sprintf("Ingress(%s/%s) is Not Found", ns, ingressName), nil
+		}
+		return false, "", err
+	}
+	return true, "", nil
+}

controllers/rollout/rollout_controller.go

@@ -0,0 +1,137 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rollout
+
+import (
+	"context"
+	"github.com/openkruise/rollouts/pkg/util"
+	"time"
+
+	kruiseappsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1"
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	apps "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+var (
+	concurrentReconciles = 3
+)
+
+// RolloutReconciler reconciles a Rollout object
+type RolloutReconciler struct {
+	client.Client
+	Scheme *runtime.Scheme
+
+	Finder *util.ControllerFinder
+}
+
+//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=rollouts,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=rollouts/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=rollouts/finalizers,verbs=update
+//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=update
+//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=rollouts.kruise.io,resources=batchreleases,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=rollouts.kruise.io,resources=batchreleases/status,verbs=get;update;patch
+
+// Reconcile is part of the main kubernetes reconciliation loop which aims to
+// move the current state of the cluster closer to the desired state.
+// TODO(user): Modify the Reconcile function to compare the state specified by
+// the Rollout object against the actual cluster state, and then
+// perform operations to make the cluster state reflect the state specified by
+// the user.
+//
+// For more details, check Reconcile and its Result here:
+// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.8.3/pkg/reconcile
+func (r *RolloutReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// Fetch the SidecarSet instance
+	rollout := &appsv1alpha1.Rollout{}
+	err := r.Get(context.TODO(), req.NamespacedName, rollout)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			// Object not found, return.  Created objects are automatically garbage collected.
+			// For additional cleanup logic use finalizers.
+			return ctrl.Result{}, nil
+		}
+		// Error reading the object - requeue the request.
+		return ctrl.Result{}, err
+	}
+
+	// handle finalizer
+	doneReconcile, retErr := r.handleFinalizer(rollout)
+	if doneReconcile {
+		return ctrl.Result{}, retErr
+	}
+	// check rollout status, and update
+	err = r.checkRolloutStatus(rollout)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	var recheckTime *time.Time
+	switch rollout.Status.Phase {
+	case appsv1alpha1.RolloutPhaseProgressing:
+		recheckTime, err = r.reconcileRolloutProgressing(rollout)
+	case appsv1alpha1.RolloutPhaseRollback:
+		recheckTime, err = r.reconcileRolloutRollback(rollout)
+	case appsv1alpha1.RolloutPhaseTerminating:
+		recheckTime, err = r.reconcileRolloutTerminating(rollout)
+	}
+	if err != nil {
+		return ctrl.Result{}, err
+	} else if recheckTime != nil {
+		return ctrl.Result{RequeueAfter: time.Until(*recheckTime)}, nil
+	}
+	return ctrl.Result{}, nil
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *RolloutReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	// Create a new controller
+	c, err := controller.New("rollout-controller", mgr, controller.Options{
+		Reconciler: r, MaxConcurrentReconciles: concurrentReconciles})
+	if err != nil {
+		return err
+	}
+	// Watch for changes to cloneset
+	if err = c.Watch(&source.Kind{Type: &kruiseappsv1alpha1.CloneSet{}}, &enqueueRequestForWorkload{reader: mgr.GetCache(), scheme: r.Scheme}); err != nil {
+		return err
+	}
+	// Watch for changes to deployment
+	if err = c.Watch(&source.Kind{Type: &apps.Deployment{}}, &enqueueRequestForWorkload{reader: mgr.GetCache(), scheme: r.Scheme}); err != nil {
+		return err
+	}
+	// Watch for changes to rollout
+	if err = c.Watch(&source.Kind{Type: &appsv1alpha1.Rollout{}}, &handler.EnqueueRequestForObject{}); err != nil {
+		return err
+	}
+	// Watch for changes to batchRelease
+	if err = c.Watch(&source.Kind{Type: &appsv1alpha1.BatchRelease{}}, &enqueueRequestForBatchRelease{reader: mgr.GetCache()}); err != nil {
+		return err
+	}
+	return nil
+}

controllers/rollout/rollout_controller_test.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rollout
+
+import (
+	kruisev1aplphal "github.com/openkruise/kruise-api/apps/v1alpha1"
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	apps "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+)
+
+var (
+	scheme *runtime.Scheme
+
+	rolloutDemo = &appsv1alpha1.Rollout{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   "rollout-demo",
+			Labels: map[string]string{},
+		},
+		Spec: appsv1alpha1.RolloutSpec{
+			ObjectRef: appsv1alpha1.ObjectRef{
+				Type: appsv1alpha1.WorkloadRefType,
+				WorkloadRef: &appsv1alpha1.WorkloadRef{
+					APIVersion: "apps/v1",
+					Kind:       "Deployment",
+					Name:       "echoserver",
+				},
+			},
+		},
+	}
+
+	deploymentDemo = &apps.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "apps/v1",
+			Kind:       "Deployment",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   "echoserver",
+			Labels: map[string]string{},
+		},
+	}
+)
+
+func init() {
+	scheme = runtime.NewScheme()
+	_ = clientgoscheme.AddToScheme(scheme)
+	_ = kruisev1aplphal.AddToScheme(scheme)
+	_ = appsv1alpha1.AddToScheme(scheme)
+}

controllers/rollout/rollouts_workload_event_handler.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2021 The SAE Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rollout
+
+import (
+	"context"
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	"github.com/openkruise/rollouts/controllers/rollout/batchrelease"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+var _ handler.EventHandler = &enqueueRequestForWorkload{}
+
+type enqueueRequestForWorkload struct {
+	reader client.Reader
+	scheme *runtime.Scheme
+}
+
+func (w *enqueueRequestForWorkload) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
+	w.handleEvent(q, evt.Object)
+}
+
+func (w *enqueueRequestForWorkload) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
+	w.handleEvent(q, evt.Object)
+}
+
+func (w *enqueueRequestForWorkload) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
+}
+
+func (w *enqueueRequestForWorkload) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
+	w.handleEvent(q, evt.ObjectNew)
+}
+
+func (w *enqueueRequestForWorkload) handleEvent(q workqueue.RateLimitingInterface, obj client.Object) {
+	key := types.NamespacedName{
+		Namespace: obj.GetNamespace(),
+		Name:      obj.GetName(),
+	}
+	kinds, _, err := w.scheme.ObjectKinds(obj)
+	if err != nil {
+		klog.Errorf("scheme ObjectKinds key(%s) failed: %s", key.String(), err.Error())
+		return
+	}
+	gvk := kinds[0]
+	rollout, err := w.getRolloutForWorkload(key, gvk)
+	if err != nil {
+		klog.Errorf("unable to get Rollout related with %s (%s/%s), err: %v", gvk.Kind, key.Namespace, key.Name, err)
+		return
+	}
+	if rollout != nil {
+		klog.Infof("workload(%s/%s) and reconcile Rollout (%s/%s)", key.Namespace, key.Name, rollout.Namespace, rollout.Name)
+		nsn := types.NamespacedName{Namespace: rollout.GetNamespace(), Name: rollout.GetName()}
+		q.Add(reconcile.Request{NamespacedName: nsn})
+	}
+}
+
+func (w *enqueueRequestForWorkload) getRolloutForWorkload(key types.NamespacedName, gvk schema.GroupVersionKind) (*appsv1alpha1.Rollout, error) {
+	rList := &appsv1alpha1.RolloutList{}
+	listOptions := &client.ListOptions{Namespace: key.Namespace}
+	if err := w.reader.List(context.TODO(), rList, listOptions); err != nil {
+		klog.Errorf("List WorkloadSpread failed: %s", err.Error())
+		return nil, err
+	}
+
+	for _, rollout := range rList.Items {
+		targetRef := rollout.Spec.ObjectRef.WorkloadRef
+		targetGV, err := schema.ParseGroupVersion(targetRef.APIVersion)
+		if err != nil {
+			klog.Errorf("failed to parse rollout(%s/%s) targetRef's group version: %s", rollout.Namespace, rollout.Name, targetRef.APIVersion)
+			continue
+		}
+
+		if targetRef.Kind == gvk.Kind && targetGV.Group == gvk.Group && targetRef.Name == key.Name {
+			return &rollout, nil
+		}
+	}
+
+	return nil, nil
+}
+
+var _ handler.EventHandler = &enqueueRequestForBatchRelease{}
+
+type enqueueRequestForBatchRelease struct {
+	reader client.Reader
+}
+
+func (w *enqueueRequestForBatchRelease) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
+}
+
+func (w *enqueueRequestForBatchRelease) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
+}
+
+func (w *enqueueRequestForBatchRelease) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
+}
+
+func (w *enqueueRequestForBatchRelease) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
+	w.handleEvent(q, evt.ObjectNew)
+}
+
+func (w *enqueueRequestForBatchRelease) handleEvent(q workqueue.RateLimitingInterface, obj client.Object) {
+	rollout := obj.GetAnnotations()[batchrelease.BatchReleaseOwnerRefAnnotation]
+	if rollout == "" {
+		return
+	}
+	klog.Infof("BatchRelease(%s/%s) and reconcile Rollout (%s)", obj.GetNamespace(), obj.GetName(), rollout)
+	nsn := types.NamespacedName{Namespace: obj.GetNamespace(), Name: rollout}
+	q.Add(reconcile.Request{NamespacedName: nsn})
+}

controllers/rollout/rollouts_workload_event_handler_test.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rollout
+
+import (
+	"context"
+	"k8s.io/client-go/util/workqueue"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"testing"
+)
+
+func TestPodEventHandler(t *testing.T) {
+	fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
+	handler := enqueueRequestForWorkload{reader: fakeClient, scheme: scheme}
+
+	err := fakeClient.Create(context.TODO(), rolloutDemo.DeepCopy())
+	if nil != err {
+		t.Fatalf("unexpected create rollout %s failed: %v", rolloutDemo.Name, err)
+	}
+
+	// create
+	createQ := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
+	createEvt := event.CreateEvent{
+		Object: deploymentDemo,
+	}
+	handler.Create(createEvt, createQ)
+	if createQ.Len() != 1 {
+		t.Errorf("unexpected create event handle queue size, expected 1 actual %d", createQ.Len())
+	}
+
+	// other namespace
+	demo1 := deploymentDemo.DeepCopy()
+	demo1.Namespace = "other-ns"
+	createEvt = event.CreateEvent{
+		Object: demo1,
+	}
+	if createQ.Len() != 1 {
+		t.Errorf("unexpected create event handle queue size, expected 1 actual %d", createQ.Len())
+	}
+}

controllers/rollout/status.go

@@ -0,0 +1,135 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rollout
+
+import (
+	"context"
+	"encoding/json"
+	"github.com/openkruise/rollouts/pkg/util"
+	"reflect"
+
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog/v2"
+)
+
+func (r *RolloutReconciler) checkRolloutStatus(rollout *appsv1alpha1.Rollout) error {
+	newStatus := *rollout.Status.DeepCopy()
+	newStatus.ObservedGeneration = rollout.GetGeneration()
+	// delete rollout CRD
+	if !rollout.DeletionTimestamp.IsZero() && newStatus.Phase != appsv1alpha1.RolloutPhaseTerminating {
+		newStatus.Phase = appsv1alpha1.RolloutPhaseTerminating
+		cond := util.NewRolloutCondition(appsv1alpha1.RolloutConditionTerminating, corev1.ConditionFalse, appsv1alpha1.TerminatingReasonInTerminating, "rollout is in terminating")
+		util.SetRolloutCondition(&newStatus, cond)
+	} else if newStatus.Phase == "" {
+		newStatus.Phase = appsv1alpha1.RolloutPhaseInitial
+	}
+	workload, err := r.Finder.GetWorkloadForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef)
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) get workload failed: %s", rollout.Namespace, rollout.Name, err.Error())
+		return err
+	} else if workload == nil && rollout.DeletionTimestamp.IsZero() {
+		resetStatus(&newStatus)
+		klog.Infof("rollout(%s/%s) workload not found, and reset status be Initial", rollout.Namespace, rollout.Name)
+	} else if workload != nil {
+		newStatus.StableRevision = workload.StableRevision
+		newStatus.CanaryRevision = workload.CanaryRevision
+	}
+	klog.Infof("rollout(%s/%s) workload(%s) StableRevision(%s) UpdateRevision(%s)",
+		rollout.Namespace, rollout.Name, rollout.Spec.ObjectRef.WorkloadRef.Name, newStatus.StableRevision, newStatus.CanaryRevision)
+
+	switch newStatus.Phase {
+	case appsv1alpha1.RolloutPhaseInitial:
+		if workload != nil {
+			klog.Infof("rollout(%s/%s) status phase from(%s) -> to(%s)", rollout.Namespace, rollout.Name, appsv1alpha1.RolloutPhaseInitial, appsv1alpha1.RolloutPhaseHealthy)
+			newStatus.Phase = appsv1alpha1.RolloutPhaseHealthy
+			newStatus.Message = "rollout is healthy"
+		}
+	case appsv1alpha1.RolloutPhaseHealthy:
+		// from healthy to rollout
+		if workload.InRolloutProgressing {
+			klog.Infof("rollout(%s/%s) status phase from(%s) -> to(%s)", rollout.Namespace, rollout.Name, appsv1alpha1.RolloutPhaseHealthy, appsv1alpha1.RolloutPhaseProgressing)
+			newStatus.Phase = appsv1alpha1.RolloutPhaseProgressing
+			// new canaryStatus
+			newStatus.CanaryStatus = &appsv1alpha1.CanaryStatus{}
+			cond := util.NewRolloutCondition(appsv1alpha1.RolloutConditionProgressing, corev1.ConditionFalse, appsv1alpha1.ProgressingReasonInitializing, "initiate rollout progressing action")
+			util.SetRolloutCondition(&newStatus, cond)
+		}
+	case appsv1alpha1.RolloutPhaseProgressing:
+		cond := util.GetRolloutCondition(newStatus, appsv1alpha1.RolloutConditionProgressing)
+		if cond == nil || cond.Reason == appsv1alpha1.ProgressingReasonSucceeded {
+			newStatus.Phase = appsv1alpha1.RolloutPhaseHealthy
+			// whether workload is in rollback phase
+		} else if workload.StableRevision == workload.CanaryRevision {
+			newStatus.Phase = appsv1alpha1.RolloutPhaseRollback
+			cond.Reason = appsv1alpha1.ProgressingReasonCanceled
+			cond.Message = "workload has been rollback"
+			util.SetRolloutCondition(&newStatus, *cond)
+			condR := util.NewRolloutCondition(appsv1alpha1.RolloutConditionRollback, corev1.ConditionFalse, appsv1alpha1.RollbackReasonInRollback, "")
+			util.SetRolloutCondition(&newStatus, condR)
+		}
+	case appsv1alpha1.RolloutPhaseRollback:
+		cond := util.GetRolloutCondition(newStatus, appsv1alpha1.RolloutConditionRollback)
+		if cond == nil || cond.Reason == appsv1alpha1.RollbackReasonCompleted {
+			newStatus.Phase = appsv1alpha1.RolloutPhaseHealthy
+		}
+	}
+	err = r.updateRolloutStatus(rollout, newStatus)
+	if err != nil {
+		klog.Errorf("update rollout(%s/%s) status failed: %s", rollout.Namespace, rollout.Name, err.Error())
+		return err
+	}
+	rollout.Status = newStatus
+	return nil
+}
+
+func (r *RolloutReconciler) updateRolloutStatus(rollout *appsv1alpha1.Rollout, newStatus appsv1alpha1.RolloutStatus) error {
+	if reflect.DeepEqual(rollout.Status, newStatus) {
+		return nil
+	}
+	rolloutClone := rollout.DeepCopy()
+	if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		if err := r.Client.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: rollout.Name}, rolloutClone); err != nil {
+			klog.Errorf("error getting updated rollout(%s/%s) from client", rollout.Namespace, rollout.Name)
+			return err
+		}
+		rolloutClone.Status = newStatus
+		rolloutClone.Status.ObservedGeneration = rolloutClone.Generation
+		if err := r.Client.Status().Update(context.TODO(), rolloutClone); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	oldBy, _ := json.Marshal(rollout.Status)
+	newBy, _ := json.Marshal(newStatus)
+	klog.Infof("rollout(%s/%s) status from(%s) -> to(%s)", rollout.Namespace, rollout.Name, string(oldBy), string(newBy))
+	return nil
+}
+
+// ResetStatus resets the status of the rollout to start from beginning
+func resetStatus(status *appsv1alpha1.RolloutStatus) {
+	status.CanaryRevision = ""
+	status.StableRevision = ""
+	util.RemoveRolloutCondition(status, appsv1alpha1.RolloutConditionProgressing)
+	status.Phase = appsv1alpha1.RolloutPhaseInitial
+	status.Message = "workload not found"
+	//status.CanaryStatus = nil
+}

controllers/rollout/trafficrouting.go

@@ -0,0 +1,248 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rollout
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/openkruise/rollouts/pkg/util"
+	"time"
+
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	"github.com/openkruise/rollouts/controllers/rollout/trafficrouting"
+	"github.com/openkruise/rollouts/controllers/rollout/trafficrouting/nginx"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog/v2"
+	utilpointer "k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) {
+	//fetch stable service
+	sName := r.rollout.Spec.Strategy.CanaryPlan.TrafficRouting.Service
+	r.stableService = &corev1.Service{}
+	err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: sName}, r.stableService)
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) get stable service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, sName, err.Error())
+		// not found, wait a moment, retry
+		if errors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	r.newStatus.CanaryStatus.CanaryService = fmt.Sprintf("%s-canary", sName)
+	// fetch canary stable
+	canaryStatus := r.newStatus.CanaryStatus
+	r.canaryService = &corev1.Service{}
+	err = r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: canaryStatus.CanaryService}, r.canaryService)
+	if err != nil && !errors.IsNotFound(err) {
+		klog.Errorf("rollout(%s/%s) get canary service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, canaryStatus.CanaryService, err.Error())
+		return false, err
+	} else if errors.IsNotFound(err) {
+		klog.Infof("rollout(%s/%s) canary service(%s) Not Found, and create it", r.rollout.Namespace, r.rollout.Name, canaryStatus.CanaryService)
+		if err = r.createCanaryService(); err != nil {
+			return false, err
+		}
+		by, _ := json.Marshal(r.canaryService)
+		klog.Infof("create rollout(%s/%s) canary service(%s) success", r.rollout.Namespace, r.rollout.Name, string(by))
+	}
+
+	// update service selector
+	// update service selector specific revision pods
+	if r.canaryService.Spec.Selector[r.podRevisionLabelKey()] != r.canaryRevision {
+		r.canaryService.Spec.Selector[r.podRevisionLabelKey()] = r.canaryRevision
+		if err = r.retryUpdateService(r.canaryService); err != nil {
+			return false, err
+		}
+		klog.Infof("add rollout(%s/%s) canary service(%s) selector(%s=%s) success",
+			r.rollout.Namespace, r.rollout.Name, r.canaryService.Name, r.podRevisionLabelKey(), r.canaryRevision)
+		// Adjust service and wait a while, just to be safe
+		return false, nil
+	}
+	if r.stableService.Spec.Selector[r.podRevisionLabelKey()] != r.stableRevision {
+		r.stableService.Spec.Selector[r.podRevisionLabelKey()] = r.stableRevision
+		if err = r.retryUpdateService(r.stableService); err != nil {
+			return false, err
+		}
+		klog.Infof("add rollout(%s/%s) stable service(%s) selector(%s=%s) success",
+			r.rollout.Namespace, r.rollout.Name, r.stableService.Name, r.podRevisionLabelKey(), r.stableRevision)
+		// Adjust service and wait a while, just to be safe
+		return false, nil
+	}
+
+	trController, err := r.newTrafficRoutingController(r)
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error())
+		return false, err
+	}
+
+	var desiredWeight int32
+	if len(r.rollout.Spec.Strategy.CanaryPlan.Steps) > 0 {
+		desiredWeight = r.rollout.Spec.Strategy.CanaryPlan.Steps[r.newStatus.CanaryStatus.CurrentStepIndex].Weight
+	}
+	verify, err := trController.VerifyTrafficRouting(desiredWeight)
+	if err != nil {
+		return false, err
+	} else if verify {
+		klog.Infof("rollout(%s/%s) do step(%d) traffic routing success", r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex)
+		return true, nil
+	}
+	return false, trController.SetRoutes(desiredWeight)
+}
+
+func (r *rolloutContext) restoreStableService() (bool, error) {
+	//fetch stable service
+	sName := r.rollout.Spec.Strategy.CanaryPlan.TrafficRouting.Service
+	r.stableService = &corev1.Service{}
+	err := r.Get(context.TODO(), client.ObjectKey{Namespace: r.rollout.Namespace, Name: sName}, r.stableService)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return true, nil
+		}
+		klog.Errorf("rollout(%s/%s) get stable service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, sName, err.Error())
+		return false, err
+	}
+
+	//restore stable service configuration，remove hash revision selector
+	if r.stableService.Spec.Selector != nil && r.stableService.Spec.Selector[r.podRevisionLabelKey()] != "" {
+		delete(r.stableService.Spec.Selector, r.podRevisionLabelKey())
+		if err := r.retryUpdateService(r.stableService); err != nil {
+			return false, err
+		}
+		// update Progressing Condition LastUpdateTime = time.Now()
+		progressingStateTransition(r.newStatus, corev1.ConditionTrue, appsv1alpha1.ProgressingReasonFinalising, "")
+		klog.Infof("remove rollout(%s/%s) stable service(%s) pod revision selector(%s) success",
+			r.rollout.Namespace, r.rollout.Name, r.stableService.Name, r.podRevisionLabelKey())
+		return false, nil
+	}
+	cond := util.GetRolloutCondition(*r.newStatus, appsv1alpha1.RolloutConditionProgressing)
+	if cond != nil {
+		// After restore stable service configuration, give the ingress provider 5 seconds to take effect
+		if verifyTime := cond.LastUpdateTime.Add(time.Second * 5); !verifyTime.Before(time.Now()) {
+			return false, nil
+		}
+		klog.Infof("rollout(%s/%s) doFinalising restore stable service(%s) success", r.rollout.Namespace, r.rollout.Name, r.stableService.Name)
+	}
+	return true, nil
+}
+
+func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) {
+	// 1. restore ingress，traffic routing stable service
+	trController, err := r.newTrafficRoutingController(r)
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error())
+		return false, err
+	}
+	done, err := trController.DoFinalising()
+	if err != nil {
+		klog.Errorf("rollout(%s/%s) DoFinalising TrafficRouting failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error())
+		return false, err
+	} else if !done {
+		return false, nil
+	}
+	klog.Infof("rollout(%s/%s) DoFinalising TrafficRouting success", r.rollout.Namespace, r.rollout.Name)
+
+	// 2. remove canary service
+	cService := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: r.rollout.Namespace,
+			Name:      r.newStatus.CanaryStatus.CanaryService,
+		},
+	}
+	err = r.Delete(context.TODO(), cService)
+	if err != nil && !errors.IsNotFound(err) {
+		klog.Errorf("rollout(%s/%s) remove canary service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, cService.Name, err.Error())
+		return false, err
+	}
+	klog.Infof("rollout(%s/%s) remove canary service(%s) success", r.rollout.Namespace, r.rollout.Name, cService.Name)
+	return true, nil
+}
+
+func (r *rolloutContext) newTrafficRoutingController(roCtx *rolloutContext) (trafficrouting.TrafficRoutingController, error) {
+	canary := roCtx.rollout.Spec.Strategy.CanaryPlan
+	switch canary.TrafficRouting.Type {
+	case appsv1alpha1.TrafficRoutingNginx:
+		gvk := schema.GroupVersionKind{Group: appsv1alpha1.GroupVersion.Group, Version: appsv1alpha1.GroupVersion.Version, Kind: "Rollout"}
+		return nginx.NewNginxTrafficRouting(r.Client, r.newStatus, nginx.NginxConfig{
+			RolloutName:   r.rollout.Name,
+			RolloutNs:     r.rollout.Namespace,
+			CanaryService: r.canaryService,
+			StableService: r.stableService,
+			TrafficConf:   r.rollout.Spec.Strategy.CanaryPlan.TrafficRouting.Nginx,
+			OwnerRef:      *metav1.NewControllerRef(r.rollout, gvk),
+		})
+	}
+
+	return nil, fmt.Errorf("TrafficRouting(%s) not support", canary.TrafficRouting.Type)
+}
+
+func (r *rolloutContext) retryUpdateService(service *corev1.Service) error {
+	obj := service.DeepCopy()
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		if err := r.Get(context.TODO(), types.NamespacedName{Name: service.Name, Namespace: service.Namespace}, obj); err != nil {
+			klog.Errorf("getting updated service(%s.%s) failed: %s", obj.Namespace, obj.Name, err.Error())
+			return err
+		}
+		obj.Spec = *service.Spec.DeepCopy()
+		return r.Update(context.TODO(), obj)
+	})
+	if err != nil {
+		klog.Errorf("update rollout(%s/%s) service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, service.Name, err.Error())
+		return err
+	}
+	return nil
+}
+
+func (r *rolloutContext) createCanaryService() error {
+	r.canaryService = &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: r.rollout.Namespace,
+			Name:      r.newStatus.CanaryStatus.CanaryService,
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion:         r.rollout.APIVersion,
+					Kind:               r.rollout.Kind,
+					Name:               r.rollout.Name,
+					UID:                r.rollout.UID,
+					Controller:         utilpointer.BoolPtr(true),
+					BlockOwnerDeletion: utilpointer.BoolPtr(true),
+				},
+			},
+		},
+		Spec: *r.stableService.Spec.DeepCopy(),
+	}
+	// set field nil
+	r.canaryService.Spec.ClusterIP = ""
+	r.canaryService.Spec.ClusterIPs = nil
+	r.canaryService.Spec.ExternalIPs = nil
+	r.canaryService.Spec.IPFamilyPolicy = nil
+	r.canaryService.Spec.IPFamilies = nil
+	r.canaryService.Spec.LoadBalancerIP = ""
+	r.canaryService.Spec.Selector[r.podRevisionLabelKey()] = r.canaryRevision
+	err := r.Create(context.TODO(), r.canaryService)
+	if err != nil && !errors.IsAlreadyExists(err) {
+		klog.Errorf("create rollout(%s/%s) canary service(%s) failed: %s", r.rollout.Namespace, r.rollout.Name, r.canaryService.Name, err.Error())
+		return err
+	}
+	return nil
+}

controllers/rollout/trafficrouting/nginx/nginx.go

@@ -0,0 +1,296 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nginx
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"time"
+
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	"github.com/openkruise/rollouts/controllers/rollout/trafficrouting"
+	corev1 "k8s.io/api/core/v1"
+	netv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	nginxIngressAnnotationDefaultPrefix = "nginx.ingress.kubernetes.io"
+
+	k8sIngressClassAnnotation = "kubernetes.io/ingress.class"
+
+	ingressTrafficRoutingState = "rollouts.kruise.io/traffic-routing-state"
+)
+
+type trafficRoutingState struct {
+	UpdateTimestamp metav1.Time `json:"updateTimestamp"`
+	Weight          int32       `json:"weight"`
+}
+
+type nginxController struct {
+	client.Client
+	//stableIngress *netv1.Ingress
+	conf      NginxConfig
+	newStatus *appsv1alpha1.RolloutStatus
+}
+
+type NginxConfig struct {
+	RolloutName   string
+	RolloutNs     string
+	CanaryService *corev1.Service
+	StableService *corev1.Service
+	TrafficConf   *appsv1alpha1.NginxTrafficRouting
+	OwnerRef      metav1.OwnerReference
+}
+
+func NewNginxTrafficRouting(client client.Client, newStatus *appsv1alpha1.RolloutStatus, albConf NginxConfig) (trafficrouting.TrafficRoutingController, error) {
+	r := &nginxController{
+		Client:    client,
+		conf:      albConf,
+		newStatus: newStatus,
+	}
+	return r, nil
+}
+
+func (r *nginxController) SetRoutes(desiredWeight int32) error {
+	stableIngress := &netv1.Ingress{}
+	err := r.Get(context.TODO(), types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.TrafficConf.Ingress}, stableIngress)
+	if err != nil {
+		return err
+	}
+	canaryIngress := &netv1.Ingress{}
+	err = r.Get(context.TODO(), types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.defaultCanaryIngressName()}, canaryIngress)
+	if err != nil && !errors.IsNotFound(err) {
+		klog.Errorf("rollout(%s/%s) get canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
+		return err
+	} else if errors.IsNotFound(err) {
+		// create canary ingress
+		canaryIngress = r.buildCanaryIngress(stableIngress, desiredWeight)
+		if err = r.Create(context.TODO(), canaryIngress); err != nil {
+			klog.Errorf("rollout(%s/%s) create canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
+			return err
+		}
+		by, _ := json.Marshal(canaryIngress)
+		klog.Infof("rollout(%s/%s) create canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, string(by))
+		return nil
+	}
+
+	currentWeight := getIngressCanaryWeight(canaryIngress)
+	if desiredWeight == currentWeight {
+		return nil
+	}
+	if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		if err := r.Client.Get(context.TODO(), types.NamespacedName{Namespace: canaryIngress.Namespace, Name: canaryIngress.Name}, canaryIngress); err != nil {
+			klog.Errorf("error getting updated ingress(%s/%s) from client", canaryIngress.Namespace, canaryIngress.Name)
+			return err
+		}
+		state := trafficRoutingState{
+			Weight:          desiredWeight,
+			UpdateTimestamp: metav1.Now(),
+		}
+		by, _ := json.Marshal(state)
+		canaryIngress.Annotations[ingressTrafficRoutingState] = string(by)
+		canaryIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = fmt.Sprintf("%d", desiredWeight)
+		if err := r.Client.Update(context.TODO(), canaryIngress); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		klog.Errorf("rollout(%s/%s) set ingress routes failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
+		return err
+	}
+	klog.Errorf("rollout(%s/%s) set ingress routes(weight:%d) success", r.conf.RolloutNs, r.conf.RolloutName, desiredWeight)
+	return nil
+}
+
+func (r *nginxController) VerifyTrafficRouting(desiredWeight int32) (bool, error) {
+	// verify set weight routing
+	canaryIngress := &netv1.Ingress{}
+	err := r.Get(context.TODO(), types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.defaultCanaryIngressName()}, canaryIngress)
+	if err != nil && !errors.IsNotFound(err) {
+		klog.Errorf("rollout(%s/%s) get canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
+		return false, err
+	} else if errors.IsNotFound(err) {
+		return false, nil
+	}
+	data, ok := canaryIngress.Annotations[ingressTrafficRoutingState]
+	if !ok {
+		return false, nil
+	}
+
+	var state *trafficRoutingState
+	if err = json.Unmarshal([]byte(data), &state); err != nil {
+		klog.Errorf("rollout(%s/%s) Unmarshal annotation[%s] failed: %s", r.conf.RolloutNs, r.conf.RolloutName, ingressTrafficRoutingState, err.Error())
+		return false, err
+	}
+	if state.Weight != desiredWeight {
+		return false, nil
+	}
+
+	// After setting up traffic routing, give the ingress provider 5 seconds to take effect
+	if verifyTime := state.UpdateTimestamp.Add(time.Second * 5); verifyTime.Before(time.Now()) {
+		klog.Infof("rollout(%s/%s) verify routes(%d) success", r.conf.RolloutNs, r.conf.RolloutName, desiredWeight)
+		return true, nil
+	}
+
+	klog.Infof("rollout(%s/%s) verify routes(%d) incomplete, and wait a moment", r.conf.RolloutNs, r.conf.RolloutName, desiredWeight)
+	return false, nil
+}
+
+func (r *nginxController) DoFinalising() (bool, error) {
+	canaryIngress := &netv1.Ingress{}
+	err := r.Get(context.TODO(), types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.defaultCanaryIngressName()}, canaryIngress)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return true, nil
+		}
+		klog.Errorf("rollout(%s/%s) get canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, r.defaultCanaryIngressName(), err.Error())
+		return false, err
+	}
+
+	var state *trafficRoutingState
+	if data, ok := canaryIngress.Annotations[ingressTrafficRoutingState]; !ok {
+		// immediate delete canary ingress
+		if err = r.Delete(context.TODO(), canaryIngress); err != nil {
+			klog.Errorf("rollout(%s/%s) remove canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error())
+			return false, err
+		}
+		klog.Infof("rollout(%s/%s) remove canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name)
+		return true, nil
+	} else {
+		if err = json.Unmarshal([]byte(data), &state); err != nil {
+			klog.Errorf("rollout(%s/%s) Unmarshal annotation[%s] failed: %s", r.conf.RolloutNs, r.conf.RolloutName, ingressTrafficRoutingState, err.Error())
+			return false, err
+		}
+	}
+
+	if state.Weight == 0 {
+		// After setting up traffic routing, give the ingress provider 5 seconds to take effect
+		if verifyTime := state.UpdateTimestamp.Add(time.Second * 5); verifyTime.After(time.Now()) {
+			klog.Errorf("rollout(%s/%s) set ingress routes(weight:0) success, and wait a moment", r.conf.RolloutNs, r.conf.RolloutName)
+			return false, nil
+		}
+		if err = r.Delete(context.TODO(), canaryIngress); err != nil {
+			klog.Errorf("rollout(%s/%s) remove canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error())
+			return false, err
+		}
+		klog.Infof("rollout(%s/%s) remove canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name)
+		return true, nil
+	}
+
+	// first, set canary ingress weight = 0
+	if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		if err = r.Client.Get(context.TODO(), types.NamespacedName{Namespace: canaryIngress.Namespace, Name: canaryIngress.Name}, canaryIngress); err != nil {
+			klog.Errorf("error getting updated ingress(%s/%s) from client", canaryIngress.Namespace, canaryIngress.Name)
+			return err
+		}
+		state = &trafficRoutingState{
+			Weight:          0,
+			UpdateTimestamp: metav1.Now(),
+		}
+		by, _ := json.Marshal(state)
+		canaryIngress.Annotations[ingressTrafficRoutingState] = string(by)
+		canaryIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = fmt.Sprintf("%d", 0)
+		if err = r.Client.Update(context.TODO(), canaryIngress); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		klog.Errorf("rollout(%s/%s) set ingress routes failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
+		return false, err
+	}
+	klog.Infof("rollout(%s/%s) set ingress routes(weight:0) success, and wait a moment", r.conf.RolloutNs, r.conf.RolloutName)
+	return false, nil
+}
+
+func (r *nginxController) buildCanaryIngress(stableIngress *netv1.Ingress, desiredWeight int32) *netv1.Ingress {
+	desiredCanaryIngress := &netv1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        r.defaultCanaryIngressName(),
+			Namespace:   stableIngress.Namespace,
+			Annotations: map[string]string{},
+		},
+		Spec: netv1.IngressSpec{
+			Rules: make([]netv1.IngressRule, 0),
+		},
+	}
+
+	// Preserve ingressClassName from stable ingress
+	if stableIngress.Spec.IngressClassName != nil {
+		desiredCanaryIngress.Spec.IngressClassName = stableIngress.Spec.IngressClassName
+	}
+
+	// Must preserve ingress.class on canary ingress, no other annotations matter
+	// See: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#canary
+	if val, ok := stableIngress.Annotations[k8sIngressClassAnnotation]; ok {
+		desiredCanaryIngress.Annotations[k8sIngressClassAnnotation] = val
+	}
+
+	// Ensure canaryIngress is owned by this Rollout for cleanup
+	desiredCanaryIngress.SetOwnerReferences([]metav1.OwnerReference{r.conf.OwnerRef})
+
+	// Copy only the rules which reference the stableService from the stableIngress to the canaryIngress
+	// and change service backend to canaryService. Rules **not** referencing the stableIngress will be ignored.
+	for ir := 0; ir < len(stableIngress.Spec.Rules); ir++ {
+		var hasStableServiceBackendRule bool
+		ingressRule := stableIngress.Spec.Rules[ir].DeepCopy()
+
+		// Update all backends pointing to the stableService to point to the canaryService now
+		for ip := 0; ip < len(ingressRule.HTTP.Paths); ip++ {
+			if ingressRule.HTTP.Paths[ip].Backend.Service.Name == r.conf.StableService.Name {
+				hasStableServiceBackendRule = true
+				ingressRule.HTTP.Paths[ip].Backend.Service.Name = r.conf.CanaryService.Name
+			}
+		}
+
+		// If this rule was using the specified stableService backend, append it to the canary Ingress spec
+		if hasStableServiceBackendRule {
+			desiredCanaryIngress.Spec.Rules = append(desiredCanaryIngress.Spec.Rules, *ingressRule)
+		}
+	}
+
+	// Always set `canary` and `canary-weight` - `canary-by-header` and `canary-by-cookie`, if set,  will always take precedence
+	desiredCanaryIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true"
+	desiredCanaryIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = fmt.Sprintf("%d", desiredWeight)
+	state := trafficRoutingState{
+		Weight:          desiredWeight,
+		UpdateTimestamp: metav1.Now(),
+	}
+	by, _ := json.Marshal(state)
+	desiredCanaryIngress.Annotations[ingressTrafficRoutingState] = string(by)
+	return desiredCanaryIngress
+}
+
+func (r *nginxController) defaultCanaryIngressName() string {
+	return fmt.Sprintf("%s-canary", r.conf.TrafficConf.Ingress)
+}
+
+func getIngressCanaryWeight(ing *netv1.Ingress) int32 {
+	weightStr, ok := ing.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]
+	if !ok {
+		return 0
+	}
+	weight, _ := strconv.Atoi(weightStr)
+	return int32(weight)
+}

controllers/rollout/trafficrouting/trafficroutingutil.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package trafficrouting
+
+// TrafficRoutingController common function across all TrafficRouting implementation
+type TrafficRoutingController interface {
+	SetRoutes(desiredWeight int32) error
+
+	VerifyTrafficRouting(desiredWeight int32) (bool, error)
+
+	DoFinalising() (bool, error)
+}

go.mod

@@ -3,8 +3,17 @@ module github.com/openkruise/rollouts
 go 1.16
 
 require (
-	k8s.io/api v0.20.2
-	k8s.io/apimachinery v0.20.2
-	k8s.io/client-go v0.20.2
+	github.com/davecgh/go-spew v1.1.1
+	github.com/onsi/ginkgo v1.16.5
+	github.com/onsi/gomega v1.17.0
+	github.com/openkruise/kruise-api v1.0.0
+	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/api v0.20.10
+	k8s.io/apiextensions-apiserver v0.20.1
+	k8s.io/apimachinery v0.20.10
+	k8s.io/client-go v0.20.10
+	k8s.io/klog/v2 v2.4.0
+	k8s.io/utils v0.0.0-20210111153108-fddb29f9d009
 	sigs.k8s.io/controller-runtime v0.8.3
+	sigs.k8s.io/yaml v1.2.0
 )

go.sum

@@ -129,10 +129,12 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -157,8 +159,10 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -166,8 +170,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -235,6 +240,7 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -277,22 +283,28 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
-github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
 github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
 github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
+github.com/openkruise/kruise-api v1.0.0 h1:ScA0LxRRNBsgbcyLhTzR9B+KpGNWsIMptzzmjTqfYQo=
+github.com/openkruise/kruise-api v1.0.0/go.mod h1:kxV/UA/vrf/hz3z+kL21c0NOawC6K1ZjaKcJFgiOwsE=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -367,6 +379,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
@@ -459,8 +472,10 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
+golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -473,6 +488,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -508,15 +524,20 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY=
 golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -563,8 +584,11 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGgVFN1DM1/wz4aoGOA5qS8=
 golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -625,8 +649,10 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -649,8 +675,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -664,18 +691,22 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
-k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw=
 k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8=
+k8s.io/api v0.20.10 h1:kAdgi1zcyenV88/uVEzS9B/fn1m4KRbmdKB0Lxl6z/M=
+k8s.io/api v0.20.10/go.mod h1:0kei3F6biGjtRQBo5dUeujq6Ji3UCh9aOSfp/THYd7I=
 k8s.io/apiextensions-apiserver v0.20.1 h1:ZrXQeslal+6zKM/HjDXLzThlz/vPSxrfK3OqL8txgVQ=
 k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk=
 k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg=
 k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
+k8s.io/apimachinery v0.20.10 h1:GcFwz5hsGgKLohcNgv8GrInk60vUdFgBXW7uOY1i1YM=
+k8s.io/apimachinery v0.20.10/go.mod h1:kQa//VOAwyVwJ2+L9kOREbsnryfsGSkSM1przND4+mw=
 k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
 k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
-k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ=
 k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE=
+k8s.io/client-go v0.20.10 h1:TgAL2pqcNWMH4eZoS9Uw0BLh2lu5a2A4pmegjp5pmsk=
+k8s.io/client-go v0.20.10/go.mod h1:fFg+aLoasv/R+xiVaWjxeqGFYltzgQcOQzkFaSRfnJ0=
 k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg=
+k8s.io/code-generator v0.20.10/go.mod h1:i6FmG+QxaLxvJsezvZp0q/gAEzzOz3U53KFibghWToU=
 k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
 k8s.io/component-base v0.20.2 h1:LMmu5I0pLtwjpp5009KLuMGFqSc2S2isGw8t1hpYKLE=
 k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0=
@@ -696,8 +727,9 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao=
 sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU=
-sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

main.go

@@ -18,20 +18,23 @@ package main
 
 import (
 	"flag"
+	"github.com/openkruise/rollouts/pkg/util"
 	"os"
 
-	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
-	// to ensure that exec-entrypoint and run can make use of them.
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
-
+	kruisev1aplphal "github.com/openkruise/kruise-api/apps/v1alpha1"
+	rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	"github.com/openkruise/rollouts/controllers/rollout"
+	"github.com/openkruise/rollouts/webhook"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
+	// to ensure that exec-entrypoint and run can make use of them.
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/klog/v2"
+	"k8s.io/klog/v2/klogr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
-
-	rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -42,7 +45,7 @@ var (
 
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
-
+	utilruntime.Must(kruisev1aplphal.AddToScheme(scheme))
 	utilruntime.Must(rolloutsv1alpha1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 }
@@ -56,13 +59,9 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	opts := zap.Options{
-		Development: true,
-	}
-	opts.BindFlags(flag.CommandLine)
+	klog.InitFlags(nil)
 	flag.Parse()
-
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+	ctrl.SetLogger(klogr.New())
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
@@ -77,7 +76,20 @@ func main() {
 		os.Exit(1)
 	}
 
+	if err = (&rollout.RolloutReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+		Finder: util.NewControllerFinder(mgr.GetClient()),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "Rollout")
+		os.Exit(1)
+	}
 	//+kubebuilder:scaffold:builder
+	setupLog.Info("setup webhook")
+	if err = webhook.SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to setup webhook")
+		os.Exit(1)
+	}
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")

pkg/util/condition.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// NewRolloutCondition creates a new rollout condition.
+func NewRolloutCondition(condType appsv1alpha1.RolloutConditionType, status corev1.ConditionStatus, reason, message string) appsv1alpha1.RolloutCondition {
+	return appsv1alpha1.RolloutCondition{
+		Type:               condType,
+		Status:             status,
+		LastUpdateTime:     metav1.Now(),
+		LastTransitionTime: metav1.Now(),
+		Reason:             reason,
+		Message:            message,
+	}
+}
+
+// GetRolloutCondition returns the condition with the provided type.
+func GetRolloutCondition(status appsv1alpha1.RolloutStatus, condType appsv1alpha1.RolloutConditionType) *appsv1alpha1.RolloutCondition {
+	for i := range status.Conditions {
+		c := status.Conditions[i]
+		if c.Type == condType {
+			return &c
+		}
+	}
+	return nil
+}
+
+// SetRolloutCondition updates the rollout to include the provided condition. If the condition that
+// we are about to add already exists and has the same status and reason, then we are not going to update
+// by returning false. Returns true if the condition was updated
+func SetRolloutCondition(status *appsv1alpha1.RolloutStatus, condition appsv1alpha1.RolloutCondition) bool {
+	currentCond := GetRolloutCondition(*status, condition.Type)
+	if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason {
+		return false
+	}
+	// Do not update lastTransitionTime if the status of the condition doesn't change.
+	if currentCond != nil && currentCond.Status == condition.Status {
+		condition.LastTransitionTime = currentCond.LastTransitionTime
+	}
+	newConditions := filterOutCondition(status.Conditions, condition.Type)
+	status.Conditions = append(newConditions, condition)
+	return true
+}
+
+// RemoveRolloutCondition removes the rollout condition with the provided type.
+func RemoveRolloutCondition(status *appsv1alpha1.RolloutStatus, condType appsv1alpha1.RolloutConditionType) {
+	status.Conditions = filterOutCondition(status.Conditions, condType)
+}
+
+// filterOutCondition returns a new slice of rollout conditions without conditions with the provided type.
+func filterOutCondition(conditions []appsv1alpha1.RolloutCondition, condType appsv1alpha1.RolloutConditionType) []appsv1alpha1.RolloutCondition {
+	var newConditions []appsv1alpha1.RolloutCondition
+	for _, c := range conditions {
+		if c.Type == condType {
+			continue
+		}
+		newConditions = append(newConditions, c)
+	}
+	return newConditions
+}

pkg/util/controller_finder.go

@@ -0,0 +1,265 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"sort"
+
+	kruiseappsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1"
+	appsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	apps "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// Workload is used to return (controller, scale, selector) fields from the
+// controller finder functions.
+type Workload struct {
+	metav1.TypeMeta
+	metav1.ObjectMeta
+
+	// workload.spec.replicas
+	WorkloadReplicas int32
+	// stable revision
+	StableRevision string
+	// canary revision
+	CanaryRevision string
+	// canary replicas
+	CanaryReplicas int32
+	// canary ready replicas
+	CanaryReadyReplicas int32
+
+	// indicate whether the workload can enter the rollout process
+	// 1. workload.Spec.Paused = true
+	// 2. the Deployment is not in a stable version (only one version of RS)
+	// 3. workload.Status.UpdateReplicas = 0
+	InRolloutProgressing bool
+}
+
+// ControllerFinderFunc is a function type that maps a pod to a list of
+// controllers and their scale.
+type ControllerFinderFunc func(namespace string, ref *appsv1alpha1.WorkloadRef) (*Workload, error)
+
+type ControllerFinder struct {
+	client.Client
+}
+
+func NewControllerFinder(c client.Client) *ControllerFinder {
+	return &ControllerFinder{
+		Client: c,
+	}
+}
+
+// +kubebuilder:rbac:groups=apps.kruise.io,resources=clonesets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=apps.kruise.io,resources=clonesets/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=apps,resources=replicasets,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=apps,resources=replicasets/status,verbs=get;update;patch
+
+func (r *ControllerFinder) GetWorkloadForRef(namespace string, ref *appsv1alpha1.WorkloadRef) (*Workload, error) {
+	for _, finder := range r.finders() {
+		scale, err := finder(namespace, ref)
+		if scale != nil || err != nil {
+			return scale, err
+		}
+	}
+	return nil, nil
+}
+
+func (r *ControllerFinder) finders() []ControllerFinderFunc {
+	return []ControllerFinderFunc{r.getKruiseCloneSet, r.getDeployment}
+}
+
+var (
+	ControllerKindDep      = apps.SchemeGroupVersion.WithKind("Deployment")
+	ControllerKruiseKindCS = kruiseappsv1alpha1.SchemeGroupVersion.WithKind("CloneSet")
+)
+
+// getKruiseCloneSet returns the kruise cloneSet referenced by the provided controllerRef.
+func (r *ControllerFinder) getKruiseCloneSet(namespace string, ref *appsv1alpha1.WorkloadRef) (*Workload, error) {
+	// This error is irreversible, so there is no need to return error
+	ok, err := verifyGroupKind(ref, ControllerKruiseKindCS.Kind, []string{ControllerKruiseKindCS.Group})
+	if !ok {
+		return nil, nil
+	}
+	cloneSet := &kruiseappsv1alpha1.CloneSet{}
+	err = r.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: ref.Name}, cloneSet)
+	if err != nil {
+		// when error is NotFound, it is ok here.
+		if errors.IsNotFound(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	var targetReplicas int32 = 1
+	if cloneSet.Spec.Replicas != nil {
+		targetReplicas = *cloneSet.Spec.Replicas
+	}
+	workload := &Workload{
+		StableRevision:      cloneSet.Status.CurrentRevision,
+		CanaryRevision:      cloneSet.Status.UpdateRevision,
+		CanaryReplicas:      cloneSet.Status.UpdatedReplicas,
+		CanaryReadyReplicas: cloneSet.Status.UpdatedReadyReplicas,
+		WorkloadReplicas:    targetReplicas,
+		ObjectMeta:          cloneSet.ObjectMeta,
+		TypeMeta:            cloneSet.TypeMeta,
+		//PodRevisionLabelKey:  CloneSetPodRevisionLabelKey,
+	}
+
+	if cloneSet.Spec.UpdateStrategy.Paused && cloneSet.Generation == cloneSet.Status.ObservedGeneration &&
+		cloneSet.Status.UpdateRevision != cloneSet.Status.CurrentRevision && cloneSet.Status.UpdatedReplicas == 0 {
+		workload.InRolloutProgressing = true
+	}
+	return workload, nil
+}
+
+// getKruiseCloneSet returns the kruise cloneSet referenced by the provided controllerRef.
+func (r *ControllerFinder) getDeployment(namespace string, ref *appsv1alpha1.WorkloadRef) (*Workload, error) {
+	// This error is irreversible, so there is no need to return error
+	ok, err := verifyGroupKind(ref, ControllerKindDep.Kind, []string{ControllerKindDep.Group})
+	if !ok {
+		return nil, nil
+	}
+	stable := &apps.Deployment{}
+	err = r.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: ref.Name}, stable)
+	if err != nil {
+		// when error is NotFound, it is ok here.
+		if errors.IsNotFound(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	workload := &Workload{
+		// deployment name
+		//CurrentRevision: obj.Name,
+		WorkloadReplicas: *stable.Spec.Replicas,
+		ObjectMeta:       stable.ObjectMeta,
+		TypeMeta:         stable.TypeMeta,
+	}
+	stableRs, err := r.GetDeploymentStableRs(stable)
+	if err != nil || stableRs == nil {
+		return workload, err
+	}
+	// stable revision
+	workload.StableRevision = stableRs.Labels[RsPodRevisionLabelKey]
+	// not in rollout progressing
+	if _, ok = workload.Annotations[InRolloutProgressingAnnotation]; !ok {
+		return workload, nil
+	}
+
+	// in rollout progressing
+	workload.InRolloutProgressing = true
+	// in rollback
+	if EqualIgnoreHash(&stableRs.Spec.Template, &stable.Spec.Template) {
+		workload.CanaryRevision = workload.StableRevision
+		return workload, nil
+	}
+
+	// canary revision
+	workload.CanaryRevision = ComputeHash(&stable.Spec.Template, nil)
+	// canary workload status
+	canary, err := r.getLatestCanaryDeployment(stable)
+	if err != nil {
+		return nil, err
+	} else if canary != nil {
+		workload.CanaryReplicas = canary.Status.Replicas
+		workload.CanaryReadyReplicas = canary.Status.ReadyReplicas
+		canaryRs, err := r.GetDeploymentStableRs(canary)
+		if err != nil || canaryRs == nil {
+			return workload, err
+		}
+		workload.CanaryRevision = canaryRs.Labels[RsPodRevisionLabelKey]
+	}
+	return workload, err
+}
+
+func (r *ControllerFinder) getLatestCanaryDeployment(stable *apps.Deployment) (*apps.Deployment, error) {
+	canaryList := &apps.DeploymentList{}
+	selector, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: map[string]string{CanaryDeploymentLabel: string(stable.UID)}})
+	err := r.List(context.TODO(), canaryList, &client.ListOptions{LabelSelector: selector})
+	if err != nil {
+		return nil, err
+	} else if len(canaryList.Items) == 0 {
+		return nil, nil
+	}
+	sort.Slice(canaryList.Items, func(i, j int) bool {
+		return canaryList.Items[j].CreationTimestamp.Before(&canaryList.Items[i].CreationTimestamp)
+	})
+	return &canaryList.Items[0], nil
+}
+
+func (r *ControllerFinder) getReplicaSetsForDeployment(obj *apps.Deployment) ([]apps.ReplicaSet, error) {
+	// List ReplicaSets owned by this Deployment
+	rsList := &apps.ReplicaSetList{}
+	selector, err := metav1.LabelSelectorAsSelector(obj.Spec.Selector)
+	if err != nil {
+		klog.Errorf("Deployment (%s/%s) get labelSelector failed: %s", obj.Namespace, obj.Name, err.Error())
+		return nil, nil
+	}
+	err = r.List(context.TODO(), rsList, &client.ListOptions{Namespace: obj.Namespace, LabelSelector: selector})
+	if err != nil {
+		return nil, err
+	}
+	rss := make([]apps.ReplicaSet, 0)
+	for i := range rsList.Items {
+		rs := rsList.Items[i]
+		if !rs.DeletionTimestamp.IsZero() || (rs.Spec.Replicas != nil && *rs.Spec.Replicas == 0) {
+			continue
+		}
+		if ref := metav1.GetControllerOf(&rs); ref != nil {
+			if ref.UID == obj.UID {
+				rss = append(rss, rs)
+			}
+		}
+	}
+	return rss, nil
+}
+
+func (r *ControllerFinder) GetDeploymentStableRs(obj *apps.Deployment) (*apps.ReplicaSet, error) {
+	rss, err := r.getReplicaSetsForDeployment(obj)
+	if err != nil {
+		return nil, err
+	}
+	if len(rss) != 1 {
+		return nil, nil
+	}
+	return &rss[0], nil
+}
+
+func verifyGroupKind(ref *appsv1alpha1.WorkloadRef, expectedKind string, expectedGroups []string) (bool, error) {
+	gv, err := schema.ParseGroupVersion(ref.APIVersion)
+	if err != nil {
+		return false, err
+	}
+
+	if ref.Kind != expectedKind {
+		return false, nil
+	}
+
+	for _, group := range expectedGroups {
+		if group == gv.Group {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}

pkg/util/util.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import "encoding/json"
+
+const (
+	InRolloutProgressingAnnotation = "rollouts.kruise.io/in-rollout-progressing"
+
+	KruiseRolloutFinalizer = "finalizers.rollouts.kruise.io"
+)
+
+// annotation[InRolloutProgressingAnnotation] = rolloutState
+type RolloutState struct {
+	RolloutName string
+	RolloutDone bool
+}
+
+func GetRolloutState(annotations map[string]string) (*RolloutState, error) {
+	if value, ok := annotations[InRolloutProgressingAnnotation]; !ok || value == "" {
+		return nil, nil
+	} else {
+		var obj *RolloutState
+		err := json.Unmarshal([]byte(value), &obj)
+		return obj, err
+	}
+}

pkg/util/workloads_utils.go

@@ -0,0 +1,124 @@
+package util
+
+import (
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"hash"
+	"hash/fnv"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/davecgh/go-spew/spew"
+	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/klog/v2"
+
+	"github.com/openkruise/rollouts/api/v1alpha1"
+)
+
+const (
+	// cloneSet pod revision label
+	CloneSetPodRevisionLabelKey = "controller-revision-hash"
+	// replicaSet pod revision label
+	RsPodRevisionLabelKey = "pod-template-hash"
+	CanaryDeploymentLabel = "rollouts.kruise.io/canary-deployment"
+	// We omit vowels from the set of available characters to reduce the chances
+	// of "bad words" being formed.
+	alphanums = "bcdfghjklmnpqrstvwxz2456789"
+
+	BatchReleaseControlAnnotation = "batchrelease.rollouts.kruise.io/control-info"
+	StashCloneSetPartition        = "batchrelease.rollouts.kruise.io/stash-partition"
+)
+
+// DeepHashObject writes specified object to hash using the spew library
+// which follows pointers and prints actual values of the nested objects
+// ensuring the hash does not change when a pointer changes.
+func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) {
+	hasher.Reset()
+	printer := spew.ConfigState{
+		Indent:         " ",
+		SortKeys:       true,
+		DisableMethods: true,
+		SpewKeys:       true,
+	}
+	printer.Fprintf(hasher, "%#v", objectToWrite)
+}
+
+// ComputeHash returns a hash value calculated from pod template and
+// a collisionCount to avoid hash collision. The hash will be safe encoded to
+// avoid bad words.
+func ComputeHash(template *v1.PodTemplateSpec, collisionCount *int32) string {
+	podTemplateSpecHasher := fnv.New32a()
+	DeepHashObject(podTemplateSpecHasher, *template)
+
+	// Add collisionCount in the hash if it exists.
+	if collisionCount != nil {
+		collisionCountBytes := make([]byte, 8)
+		binary.LittleEndian.PutUint32(collisionCountBytes, uint32(*collisionCount))
+		podTemplateSpecHasher.Write(collisionCountBytes)
+	}
+
+	return SafeEncodeString(fmt.Sprint(podTemplateSpecHasher.Sum32()))
+}
+
+// SafeEncodeString encodes s using the same characters as rand.String. This reduces the chances of bad words and
+// ensures that strings generated from hash functions appear consistent throughout the API.
+func SafeEncodeString(s string) string {
+	r := make([]byte, len(s))
+	for i, b := range []rune(s) {
+		r[i] = alphanums[(int(b) % len(alphanums))]
+	}
+	return string(r)
+}
+
+func IsControlledBy(object, owner metav1.Object) bool {
+	controlInfo, controlled := object.GetAnnotations()[BatchReleaseControlAnnotation]
+	if !controlled {
+		return false
+	}
+
+	o := &metav1.OwnerReference{}
+	if err := json.Unmarshal([]byte(controlInfo), o); err != nil {
+		return false
+	}
+
+	return o.UID == owner.GetUID()
+}
+
+func CalculateNewBatchTarget(rolloutSpec *v1alpha1.ReleasePlan, workloadReplicas, currentBatch int) int {
+	batchSize, _ := intstr.GetValueFromIntOrPercent(&rolloutSpec.Batches[currentBatch].CanaryReplicas, workloadReplicas, true)
+	if batchSize > workloadReplicas {
+		klog.Warningf("releasePlan has wrong batch replicas, batches[%d].replicas %v is more than workload.replicas %v", currentBatch, batchSize, workloadReplicas)
+		batchSize = workloadReplicas
+	} else if batchSize < 0 {
+		klog.Warningf("releasePlan has wrong batch replicas, batches[%d].replicas %v is less than 0 %v", currentBatch, batchSize)
+		batchSize = 0
+	}
+
+	klog.V(3).InfoS("calculated the number of new pod size", "current batch", currentBatch,
+		"new pod target", batchSize)
+	return batchSize
+}
+
+func EqualIgnoreHash(template1, template2 *v1.PodTemplateSpec) bool {
+	t1Copy := template1.DeepCopy()
+	t2Copy := template2.DeepCopy()
+	// Remove hash labels from template.Labels before comparing
+	delete(t1Copy.Labels, apps.DefaultDeploymentUniqueLabelKey)
+	delete(t2Copy.Labels, apps.DefaultDeploymentUniqueLabelKey)
+	return apiequality.Semantic.DeepEqual(t1Copy, t2Copy)
+}
+
+func PatchFinalizer(c client.Client, object client.Object, finalizers []string) error {
+	patchByte, _ := json.Marshal(map[string]interface{}{
+		"metadata": map[string]interface{}{
+			"finalizers": finalizers,
+		},
+	})
+	return c.Patch(context.TODO(), object, client.RawPatch(types.MergePatchType, patchByte))
+}

test/e2e/rollout_test.go

@@ -0,0 +1,266 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"github.com/openkruise/rollouts/pkg/util"
+	netv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/util/retry"
+	"sort"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	kruiseappsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1"
+	rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	nginxIngressAnnotationDefaultPrefix = "nginx.ingress.kubernetes.io"
+)
+
+var _ = SIGDescribe("Test Rollout Controller", func() {
+	var namespace string
+
+	CreateObject := func(object client.Object, options ...client.CreateOption) {
+		object.SetNamespace(namespace)
+		Expect(k8sClient.Create(context.TODO(), object)).NotTo(HaveOccurred())
+	}
+
+	GetObject := func(name string, object client.Object) error {
+		key := types.NamespacedName{Namespace: namespace, Name: name}
+		return k8sClient.Get(context.TODO(), key, object)
+	}
+
+	UpdateDeployment := func(object *apps.Deployment) *apps.Deployment {
+		var clone *apps.Deployment
+		Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			clone = &apps.Deployment{}
+			err := GetObject(object.Name, clone)
+			if err != nil {
+				return err
+			}
+			clone.Spec = *object.Spec.DeepCopy()
+			return k8sClient.Update(context.TODO(), clone)
+		})).NotTo(HaveOccurred())
+
+		return clone
+	}
+
+	ResumeRolloutCanary := func(name string) {
+		Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			clone := &rolloutsv1alpha1.Rollout{}
+			err := GetObject(name, clone)
+			if err != nil {
+				return err
+			}
+			if clone.Status.CanaryStatus.CurrentStepState != rolloutsv1alpha1.CanaryStepStatePaused {
+				return nil
+			}
+			clone.Status.CanaryStatus.CurrentStepState = rolloutsv1alpha1.CanaryStepStateCompleted
+			return k8sClient.Status().Update(context.TODO(), clone)
+		})).NotTo(HaveOccurred())
+	}
+
+	WaitDeploymentAllPodsReady := func(deployment *apps.Deployment) {
+		Eventually(func() bool {
+			clone := &apps.Deployment{}
+			Expect(GetObject(deployment.Name, clone)).NotTo(HaveOccurred())
+			return clone.Status.ObservedGeneration == clone.Generation && clone.Status.Replicas == clone.Status.ReadyReplicas
+		}, 20*time.Minute, time.Second).Should(BeTrue())
+	}
+
+	WaitRolloutCanaryStepPaused := func(name string, stepIndex int32) {
+		Eventually(func() bool {
+			clone := &rolloutsv1alpha1.Rollout{}
+			Expect(GetObject(name, clone)).NotTo(HaveOccurred())
+			return clone.Status.CanaryStatus.CurrentStepIndex == stepIndex && clone.Status.CanaryStatus.CurrentStepState == rolloutsv1alpha1.CanaryStepStatePaused
+		}, 20*time.Minute, time.Second).Should(BeTrue())
+	}
+
+	WaitRolloutStatusPhase := func(name string, phase rolloutsv1alpha1.RolloutPhase) {
+		Eventually(func() bool {
+			clone := &rolloutsv1alpha1.Rollout{}
+			Expect(GetObject(name, clone)).NotTo(HaveOccurred())
+			return clone.Status.Phase == phase
+		}, 20*time.Minute, time.Second).Should(BeTrue())
+	}
+
+	GetCanaryDeployment := func(stable *apps.Deployment) (*apps.Deployment, error) {
+		canaryList := &apps.DeploymentList{}
+		selector, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: map[string]string{util.CanaryDeploymentLabel: string(stable.UID)}})
+		err := k8sClient.List(context.TODO(), canaryList, &client.ListOptions{LabelSelector: selector})
+		if err != nil {
+			return nil, err
+		} else if len(canaryList.Items) == 0 {
+			return nil, nil
+		}
+		sort.Slice(canaryList.Items, func(i, j int) bool {
+			return canaryList.Items[j].CreationTimestamp.Before(&canaryList.Items[i].CreationTimestamp)
+		})
+		return &canaryList.Items[0], nil
+	}
+
+	BeforeEach(func() {
+		namespace = randomNamespaceName("rollout")
+		ns := v1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: namespace,
+			},
+		}
+		Expect(k8sClient.Create(context.TODO(), &ns)).Should(SatisfyAny(BeNil()))
+	})
+
+	AfterEach(func() {
+		By("[TEST] Clean up resources after an integration test")
+		k8sClient.DeleteAllOf(context.TODO(), &apps.Deployment{}, client.InNamespace(namespace))
+		k8sClient.DeleteAllOf(context.TODO(), &kruiseappsv1alpha1.CloneSet{}, client.InNamespace(namespace))
+		k8sClient.DeleteAllOf(context.TODO(), &rolloutsv1alpha1.BatchRelease{}, client.InNamespace(namespace))
+		k8sClient.DeleteAllOf(context.TODO(), &rolloutsv1alpha1.Rollout{}, client.InNamespace(namespace))
+		Expect(k8sClient.Delete(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed())
+	})
+
+	KruiseDescribe("Deployment rollout canary nginx", func() {
+
+		It("V1->V2: Percentage, 20%,40%,60%,80%,100% Succeeded", func() {
+			By("Creating Rollout...")
+			rollout := &rolloutsv1alpha1.Rollout{}
+			Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred())
+			CreateObject(rollout)
+
+			By("Creating workload and waiting for all pods ready...")
+			// service
+			service := &v1.Service{}
+			Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred())
+			CreateObject(service)
+			// ingress
+			ingress := &netv1.Ingress{}
+			Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred())
+			CreateObject(ingress)
+			// workload
+			workload := &apps.Deployment{}
+			Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred())
+			CreateObject(workload)
+			WaitDeploymentAllPodsReady(workload)
+			time.Sleep(time.Second * 5)
+
+			// check rollout status
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy))
+			Expect(rollout.Status.StableRevision).ShouldNot(Equal(""))
+			stableRevision := rollout.Status.StableRevision
+			By("check rollout status & paused success")
+
+			// v1 -> v2, start rollout action
+			newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"})
+			workload.Spec.Template.Spec.Containers[0].Env = newEnvs
+			UpdateDeployment(workload)
+			By("Update deployment env NODE_NAME from(version1) -> to(version2)")
+			time.Sleep(time.Second * 2)
+
+			// check workload status & paused
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			Expect(workload.Spec.Paused).Should(BeTrue())
+			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0))
+			Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas))
+			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
+			By("check deployment status & paused success")
+
+			// wait step 0 complete
+			WaitRolloutCanaryStepPaused(rollout.Name, 0)
+			// check rollout status
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing))
+			Expect(rollout.Status.StableRevision).Should(Equal(stableRevision))
+			Expect(rollout.Status.CanaryRevision).ShouldNot(Equal(""))
+			canaryRevision := rollout.Status.CanaryRevision
+			Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 0))
+			// check stable, canary service & ingress
+			// stable service
+			Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
+			sRevision := stableRevision[strings.LastIndex(stableRevision, "-")+1:]
+			Expect(service.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(sRevision))
+			//canary service
+			cService := &v1.Service{}
+			Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred())
+			cRevision := canaryRevision[strings.LastIndex(canaryRevision, "-")+1:]
+			Expect(cService.Spec.Selector[util.RsPodRevisionLabelKey]).Should(Equal(cRevision))
+			// canary ingress
+			cIngress := &netv1.Ingress{}
+			Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred())
+			Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true"))
+			Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", rollout.Spec.Strategy.CanaryPlan.Steps[0].Weight)))
+			// canary deployment
+			cWorkload, err := GetCanaryDeployment(workload)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 1))
+			Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 1))
+
+			// resume rollout canary
+			ResumeRolloutCanary(rollout.Name)
+			By("check rollout canary status success, resume rollout, and wait rollout canary complete")
+			WaitRolloutCanaryStepPaused(rollout.Name, 4)
+
+			// check stable, canary service & ingress
+			// canary ingress
+			cIngress = &netv1.Ingress{}
+			Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred())
+			Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(fmt.Sprintf("%d", rollout.Spec.Strategy.CanaryPlan.Steps[4].Weight)))
+			// canary deployment
+			cWorkload, err = GetCanaryDeployment(workload)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 5))
+			Expect(cWorkload.Status.AvailableReplicas).Should(BeNumerically("==", 5))
+			// stable deployment
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			Expect(workload.Spec.Paused).Should(BeTrue())
+			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0))
+			Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas))
+			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
+
+			// resume rollout
+			ResumeRolloutCanary(rollout.Name)
+			By("check rollout canary status success, resume rollout, and wait rollout canary complete")
+			WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy)
+
+			// check progressing succeed
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing)
+			Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded))
+		})
+	})
+})
+
+func mergeEnvVar(original []v1.EnvVar, add v1.EnvVar) []v1.EnvVar {
+	newEnvs := make([]v1.EnvVar, 0)
+	for _, env := range original {
+		if add.Name == env.Name {
+			continue
+		}
+		newEnvs = append(newEnvs, env)
+	}
+	newEnvs = append(newEnvs, add)
+	return newEnvs
+}

test/e2e/suite_test.go

@@ -0,0 +1,108 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/yaml"
+
+	// +kubebuilder:scaffold:imports
+	kruise "github.com/openkruise/kruise-api/apps/v1alpha1"
+	rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+)
+
+var k8sClient client.Client
+var scheme = runtime.NewScheme()
+
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+
+	RunSpecsWithDefaultAndCustomReporters(t,
+		"Kruise Rollout Resource Controller Suite",
+		[]Reporter{printer.NewlineReporter{}})
+}
+
+var _ = BeforeSuite(func(done Done) {
+	By("Bootstrapping test environment")
+	rand.Seed(time.Now().UnixNano())
+	logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(GinkgoWriter)))
+	err := clientgoscheme.AddToScheme(scheme)
+	Expect(err).Should(BeNil())
+	err = rolloutsv1alpha1.AddToScheme(scheme)
+	Expect(err).Should(BeNil())
+	err = crdv1.AddToScheme(scheme)
+	Expect(err).Should(BeNil())
+	err = kruise.AddToScheme(scheme)
+	Expect(err).Should(BeNil())
+	By("Setting up kubernetes client")
+	k8sClient, err = client.New(config.GetConfigOrDie(), client.Options{Scheme: scheme})
+	if err != nil {
+		logf.Log.Error(err, "failed to create k8sClient")
+		Fail("setup failed")
+	}
+	close(done)
+	By("Finished setting up test environment")
+}, 300)
+
+var _ = AfterSuite(func() {
+})
+
+// RequestReconcileNow will trigger an immediate reconciliation on K8s object.
+// Some test cases may fail for timeout to wait a scheduled reconciliation.
+// This is a workaround to avoid long-time wait before next scheduled
+// reconciliation.
+func RequestReconcileNow(ctx context.Context, o client.Object) {
+	oCopy := o.DeepCopyObject()
+	oMeta, ok := oCopy.(metav1.Object)
+	Expect(ok).Should(BeTrue())
+	oMeta.SetAnnotations(map[string]string{
+		"app.oam.dev/requestreconcile": time.Now().String(),
+	})
+	oMeta.SetResourceVersion("")
+	By(fmt.Sprintf("Requset reconcile %q now", oMeta.GetName()))
+	Expect(k8sClient.Patch(ctx, oCopy.(client.Object), client.Merge)).Should(Succeed())
+}
+
+// ReadYamlToObject will read a yaml K8s object to runtime.Object
+func ReadYamlToObject(path string, object runtime.Object) error {
+	data, err := os.ReadFile(filepath.Clean(path))
+	if err != nil {
+		return err
+	}
+	return yaml.Unmarshal(data, object)
+}
+
+// randomNamespaceName generates a random name based on the basic name.
+// Running each ginkgo case in a new namespace with a random name can avoid
+// waiting a long time to GC namespace.
+func randomNamespaceName(basic string) string {
+	return fmt.Sprintf("%s-%s", basic, strconv.FormatInt(rand.Int63(), 16))
+}
+
+// SIGDescribe describes SIG information
+func SIGDescribe(text string, body func()) bool {
+	return Describe("[apps] "+text, body)
+}
+
+// KruiseDescribe is a wrapper function for ginkgo describe.  Adds namespacing.
+func KruiseDescribe(text string, body func()) bool {
+	return Describe("[kruise.io] "+text, body)
+}

test/e2e/test_data/batchrelease/cloneset.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps.kruise.io/v1alpha1
+kind: CloneSet
+metadata:
+  labels:
+    app: busybox
+  name: sample
+spec:
+  replicas: 5
+  selector:
+    matchLabels:
+      app: busybox
+  template:
+    metadata:
+      labels:
+        app: busybox
+    spec:
+      containers:
+        - name: busybox
+          image: busybox:1.32
+          imagePullPolicy: IfNotPresent
+          command: ["bin/sh", "-c", "sleep 10000000"]
+          resources:
+            limits:
+              memory: "10Mi"
+              cpu: "50m"

test/e2e/test_data/batchrelease/cloneset_number_100.yaml

@@ -0,0 +1,21 @@
+apiVersion: rollouts.kruise.io/v1alpha1
+kind: BatchRelease
+metadata:
+  name: release-cloneset-100
+spec:
+  targetReference:
+    apiVersion: apps.kruise.io/v1alpha1
+    kind: CloneSet
+    name: sample
+  releasePlan:
+    batches:
+      - canaryReplicas: 1
+        pauseSeconds: 30
+      - canaryReplicas: 2
+        pauseSeconds: 30
+      - canaryReplicas: 3
+        pauseSeconds: 30
+      - canaryReplicas: 5
+        pauseSeconds: 30
+
+

test/e2e/test_data/batchrelease/cloneset_percentage_100.yaml

@@ -0,0 +1,21 @@
+apiVersion: rollouts.kruise.io/v1alpha1
+kind: BatchRelease
+metadata:
+  name: release-cloneset-100
+spec:
+  targetReference:
+    apiVersion: apps.kruise.io/v1alpha1
+    kind: CloneSet
+    name: sample
+  releasePlan:
+    batches:
+      - canaryReplicas: 20%
+        pauseSeconds: 30
+      - canaryReplicas: 40%
+        pauseSeconds: 30
+      - canaryReplicas: 60%
+        pauseSeconds: 30
+      - canaryReplicas: 100%
+        pauseSeconds: 30
+
+

test/e2e/test_data/batchrelease/cloneset_percentage_50.yaml

@@ -0,0 +1,20 @@
+apiVersion: rollouts.kruise.io/v1alpha1
+kind: BatchRelease
+metadata:
+  name: release-cloneset-50
+spec:
+  targetReference:
+    apiVersion: apps.kruise.io/v1alpha1
+    kind: CloneSet
+    name: sample
+  releasePlan:
+    batches:
+      - canaryReplicas: 20%
+        pauseSeconds: 30
+      - canaryReplicas: 40%
+        pauseSeconds: 30
+      - canaryReplicas: 50%
+        pauseSeconds: 100
+
+
+

test/e2e/test_data/batchrelease/deployment.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sample
+  labels:
+    app: busybox
+spec:
+  replicas: 5
+  selector:
+    matchLabels:
+      app: busybox
+  template:
+    metadata:
+      labels:
+        app: busybox
+    spec:
+      containers:
+        - name: busybox
+          image: busybox:1.32
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "-c", "sleep 10000"]
+          resources:
+            limits:
+              memory: "10Mi"
+              cpu: "50m"

test/e2e/test_data/batchrelease/deployment_number_100.yaml

@@ -0,0 +1,21 @@
+apiVersion: rollouts.kruise.io/v1alpha1
+kind: BatchRelease
+metadata:
+  name: release-deployment-number-100
+spec:
+  targetReference:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: sample
+  releasePlan:
+    batches:
+      - canaryReplicas: 1
+        pauseSeconds: 30
+      - canaryReplicas: 2
+        pauseSeconds: 30
+      - canaryReplicas: 3
+        pauseSeconds: 30
+      - canaryReplicas: 5
+        pauseSeconds: 30
+
+

test/e2e/test_data/batchrelease/deployment_percentage_100.yaml

@@ -0,0 +1,19 @@
+apiVersion: rollouts.kruise.io/v1alpha1
+kind: BatchRelease
+metadata:
+  name: release-deployment-percentage-100
+spec:
+  targetReference:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: sample
+  releasePlan:
+    batches:
+      - canaryReplicas: 20%
+        pauseSeconds: 30
+      - canaryReplicas: 40%
+        pauseSeconds: 30
+      - canaryReplicas: 60%
+        pauseSeconds: 30
+      - canaryReplicas: 100%
+        pauseSeconds: 30

test/e2e/test_data/batchrelease/deployment_percentage_50.yaml

@@ -0,0 +1,20 @@
+apiVersion: rollouts.kruise.io/v1alpha1
+kind: BatchRelease
+metadata:
+  name: release-deployment-percentage-50
+spec:
+  targetReference:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: sample
+  releasePlan:
+    batches:
+      - canaryReplicas: 20%
+        pauseSeconds: 30
+      - canaryReplicas: 40%
+        pauseSeconds: 30
+      - canaryReplicas: 50%
+        pauseSeconds: 100
+
+
+

test/e2e/test_data/rollout/deployment.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: echoserver
+  labels:
+    app: echoserver
+spec:
+  replicas: 5
+  selector:
+    matchLabels:
+      app: echoserver
+  template:
+    metadata:
+      labels:
+        app: echoserver
+    spec:
+      containers:
+        - name: echoserver
+          image: registry.aliyuncs.com/google_containers/echoserver:1.10
+          ports:
+            - containerPort: 8080
+          env:
+            - name: NODE_NAME
+              value: version1
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: VERSION
+              value: v1

test/e2e/test_data/rollout/nginx_ingress.yaml

@@ -0,0 +1,18 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: echoserver
+  annotations:
+    kubernetes.io/ingress.class: nginx
+spec:
+  rules:
+    - host: echoserver.example.com
+      http:
+        paths:
+          - backend:
+              service:
+                name: echoserver
+                port:
+                  number: 80
+            path: /apis/echo
+            pathType: Exact

test/e2e/test_data/rollout/rollout_canary_base.yaml

@@ -0,0 +1,29 @@
+apiVersion: rollouts.kruise.io/v1alpha1
+kind: Rollout
+metadata:
+  name: rollouts-demo
+spec:
+  objectRef:
+    type: workloadRef
+    workloadRef:
+      apiVersion: apps/v1
+      kind: Deployment
+      name: echoserver
+  strategy:
+    canaryPlan:
+      steps:
+        - weight: 20
+          pause: {}
+        - weight: 40
+          pause: {duration: 5}
+        - weight: 60
+          pause: {duration: 5}
+        - weight: 80
+          pause: {duration: 5}
+        - weight: 100
+          pause: {}
+      trafficRouting:
+        service: echoserver
+        type: nginx
+        nginx:
+          ingress: echoserver

test/e2e/test_data/rollout/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: echoserver
+  labels:
+    app: echoserver
+spec:
+  ports:
+    - port: 80
+      targetPort: 8080
+      protocol: TCP
+      name: http
+  selector:
+    app: echoserver

test/images/image_utils.go

@@ -0,0 +1,319 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package images
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+// RegistryList holds public and private image registries
+type RegistryList struct {
+	GcAuthenticatedRegistry string `yaml:"gcAuthenticatedRegistry"`
+	DockerLibraryRegistry   string `yaml:"dockerLibraryRegistry"`
+	DockerGluster           string `yaml:"dockerGluster"`
+	E2eRegistry             string `yaml:"e2eRegistry"`
+	E2eVolumeRegistry       string `yaml:"e2eVolumeRegistry"`
+	PromoterE2eRegistry     string `yaml:"promoterE2eRegistry"`
+	BuildImageRegistry      string `yaml:"buildImageRegistry"`
+	InvalidRegistry         string `yaml:"invalidRegistry"`
+	GcRegistry              string `yaml:"gcRegistry"`
+	SigStorageRegistry      string `yaml:"sigStorageRegistry"`
+	GcrReleaseRegistry      string `yaml:"gcrReleaseRegistry"`
+	PrivateRegistry         string `yaml:"privateRegistry"`
+	SampleRegistry          string `yaml:"sampleRegistry"`
+}
+
+// Config holds an images registry, name, and version
+type Config struct {
+	registry string
+	name     string
+	version  string
+}
+
+// SetRegistry sets an image registry in a Config struct
+func (i *Config) SetRegistry(registry string) {
+	i.registry = registry
+}
+
+// SetName sets an image name in a Config struct
+func (i *Config) SetName(name string) {
+	i.name = name
+}
+
+// SetVersion sets an image version in a Config struct
+func (i *Config) SetVersion(version string) {
+	i.version = version
+}
+
+func initReg() RegistryList {
+	registry := RegistryList{
+		GcAuthenticatedRegistry: "gcr.io/authenticated-image-pulling",
+		DockerLibraryRegistry:   "docker.io/library",
+		DockerGluster:           "docker.io/gluster",
+		E2eRegistry:             "gcr.io/kubernetes-e2e-test-images",
+		E2eVolumeRegistry:       "gcr.io/kubernetes-e2e-test-images/volume",
+		PromoterE2eRegistry:     "k8s.gcr.io/e2e-test-images",
+		BuildImageRegistry:      "k8s.gcr.io/build-image",
+		InvalidRegistry:         "invalid.com/invalid",
+		GcRegistry:              "k8s.gcr.io",
+		SigStorageRegistry:      "k8s.gcr.io/sig-storage",
+		GcrReleaseRegistry:      "gcr.io/gke-release",
+		PrivateRegistry:         "gcr.io/k8s-authenticated-test",
+		SampleRegistry:          "gcr.io/google-samples",
+	}
+	repoList := os.Getenv("KUBE_TEST_REPO_LIST")
+	if repoList == "" {
+		return registry
+	}
+
+	fileContent, err := ioutil.ReadFile(repoList)
+	if err != nil {
+		panic(fmt.Errorf("Error reading '%v' file contents: %v", repoList, err))
+	}
+
+	err = yaml.Unmarshal(fileContent, &registry)
+	if err != nil {
+		panic(fmt.Errorf("Error unmarshalling '%v' YAML file: %v", repoList, err))
+	}
+	return registry
+}
+
+var (
+	registry                = initReg()
+	dockerLibraryRegistry   = registry.DockerLibraryRegistry
+	dockerGluster           = registry.DockerGluster
+	e2eRegistry             = registry.E2eRegistry
+	e2eVolumeRegistry       = registry.E2eVolumeRegistry
+	promoterE2eRegistry     = registry.PromoterE2eRegistry
+	buildImageRegistry      = registry.BuildImageRegistry
+	gcAuthenticatedRegistry = registry.GcAuthenticatedRegistry
+	gcRegistry              = registry.GcRegistry
+	sigStorageRegistry      = registry.SigStorageRegistry
+	gcrReleaseRegistry      = registry.GcrReleaseRegistry
+	invalidRegistry         = registry.InvalidRegistry
+	// PrivateRegistry is an image repository that requires authentication
+	PrivateRegistry = registry.PrivateRegistry
+	sampleRegistry  = registry.SampleRegistry
+
+	// Preconfigured image configs
+	imageConfigs = initImageConfigs()
+)
+
+const (
+	// None is to be used for unset/default images
+	None = iota
+	// Agnhost image
+	Agnhost
+	// AgnhostPrivate image
+	AgnhostPrivate
+	// APIServer image
+	APIServer
+	// AppArmorLoader image
+	AppArmorLoader
+	// AuthenticatedAlpine image
+	AuthenticatedAlpine
+	// AuthenticatedWindowsNanoServer image
+	AuthenticatedWindowsNanoServer
+	// BusyBox image
+	BusyBox
+	// CheckMetadataConcealment image
+	CheckMetadataConcealment
+	// CudaVectorAdd image
+	CudaVectorAdd
+	// CudaVectorAdd2 image
+	CudaVectorAdd2
+	// DebianIptables Image
+	DebianIptables
+	// EchoServer image
+	EchoServer
+	// Etcd image
+	Etcd
+	// GlusterDynamicProvisioner image
+	GlusterDynamicProvisioner
+	// Httpd image
+	Httpd
+	// HttpdNew image
+	HttpdNew
+	// InvalidRegistryImage image
+	InvalidRegistryImage
+	// IpcUtils image
+	IpcUtils
+	// JessieDnsutils image
+	JessieDnsutils
+	// Kitten image
+	Kitten
+	// Nautilus image
+	Nautilus
+	// NFSProvisioner image
+	NFSProvisioner
+	// Nginx image
+	Nginx
+	// NginxNew image
+	NginxNew
+	// Nonewprivs image
+	Nonewprivs
+	// NonRoot runs with a default user of 1234
+	NonRoot
+	// Pause - when these values are updated, also update cmd/kubelet/app/options/container_runtime.go
+	// Pause image
+	Pause
+	// Perl image
+	Perl
+	// PrometheusDummyExporter image
+	PrometheusDummyExporter
+	// PrometheusToSd image
+	PrometheusToSd
+	// Redis image
+	Redis
+	// RegressionIssue74839 image
+	RegressionIssue74839
+	// ResourceConsumer image
+	ResourceConsumer
+	// SdDummyExporter image
+	SdDummyExporter
+	// VolumeNFSServer image
+	VolumeNFSServer
+	// VolumeISCSIServer image
+	VolumeISCSIServer
+	// VolumeGlusterServer image
+	VolumeGlusterServer
+	// VolumeRBDServer image
+	VolumeRBDServer
+
+	BusyBoxV1
+	BusyBoxV2
+	BusyBoxV3
+	BusyBoxV4
+	FailedImage
+)
+
+func initImageConfigs() map[int]Config {
+	configs := map[int]Config{}
+	configs[Agnhost] = Config{promoterE2eRegistry, "agnhost", "2.21"}
+	configs[AgnhostPrivate] = Config{PrivateRegistry, "agnhost", "2.6"}
+	configs[AuthenticatedAlpine] = Config{gcAuthenticatedRegistry, "alpine", "3.7"}
+	configs[AuthenticatedWindowsNanoServer] = Config{gcAuthenticatedRegistry, "windows-nanoserver", "v1"}
+	configs[APIServer] = Config{e2eRegistry, "sample-apiserver", "1.17"}
+	configs[AppArmorLoader] = Config{e2eRegistry, "apparmor-loader", "1.0"}
+	configs[BusyBox] = Config{dockerLibraryRegistry, "busybox", "1.29"}
+	configs[CheckMetadataConcealment] = Config{promoterE2eRegistry, "metadata-concealment", "1.6"}
+	configs[CudaVectorAdd] = Config{e2eRegistry, "cuda-vector-add", "1.0"}
+	configs[CudaVectorAdd2] = Config{e2eRegistry, "cuda-vector-add", "2.0"}
+	configs[DebianIptables] = Config{buildImageRegistry, "debian-iptables", "buster-v1.6.5"}
+	configs[EchoServer] = Config{e2eRegistry, "echoserver", "2.2"}
+	configs[Etcd] = Config{gcRegistry, "etcd", "3.4.13-0"}
+	configs[GlusterDynamicProvisioner] = Config{dockerGluster, "glusterdynamic-provisioner", "v1.0"}
+	configs[Httpd] = Config{dockerLibraryRegistry, "httpd", "2.4.38-alpine"}
+	configs[HttpdNew] = Config{dockerLibraryRegistry, "httpd", "2.4.39-alpine"}
+	configs[InvalidRegistryImage] = Config{invalidRegistry, "alpine", "3.1"}
+	configs[IpcUtils] = Config{e2eRegistry, "ipc-utils", "1.0"}
+	configs[JessieDnsutils] = Config{e2eRegistry, "jessie-dnsutils", "1.0"}
+	configs[Kitten] = Config{e2eRegistry, "kitten", "1.0"}
+	configs[Nautilus] = Config{e2eRegistry, "nautilus", "1.0"}
+	configs[NFSProvisioner] = Config{sigStorageRegistry, "nfs-provisioner", "v2.2.2"}
+	configs[Nginx] = Config{dockerLibraryRegistry, "nginx", "1.14-alpine"}
+	configs[NginxNew] = Config{dockerLibraryRegistry, "nginx", "1.15-alpine"}
+	configs[Nonewprivs] = Config{e2eRegistry, "nonewprivs", "1.0"}
+	configs[NonRoot] = Config{e2eRegistry, "nonroot", "1.0"}
+	// Pause - when these values are updated, also update cmd/kubelet/app/options/container_runtime.go
+	configs[Pause] = Config{gcRegistry, "pause", "3.2"}
+	configs[Perl] = Config{dockerLibraryRegistry, "perl", "5.26"}
+	configs[PrometheusDummyExporter] = Config{gcRegistry, "prometheus-dummy-exporter", "v0.1.0"}
+	configs[PrometheusToSd] = Config{gcRegistry, "prometheus-to-sd", "v0.5.0"}
+	configs[Redis] = Config{dockerLibraryRegistry, "redis", "5.0.5-alpine"}
+	configs[RegressionIssue74839] = Config{e2eRegistry, "regression-issue-74839-amd64", "1.0"}
+	configs[ResourceConsumer] = Config{e2eRegistry, "resource-consumer", "1.5"}
+	configs[SdDummyExporter] = Config{gcRegistry, "sd-dummy-exporter", "v0.2.0"}
+	configs[VolumeNFSServer] = Config{e2eVolumeRegistry, "nfs", "1.0"}
+	configs[VolumeISCSIServer] = Config{e2eVolumeRegistry, "iscsi", "2.0"}
+	configs[VolumeGlusterServer] = Config{e2eVolumeRegistry, "gluster", "1.0"}
+	configs[VolumeRBDServer] = Config{e2eVolumeRegistry, "rbd", "1.0.1"}
+	// Rollout - when test kruise rollout e2e cases
+	configs[BusyBoxV1] = Config{dockerLibraryRegistry, "busybox", "1.29"}
+	configs[BusyBoxV2] = Config{dockerLibraryRegistry, "busybox", "1.30"}
+	configs[BusyBoxV3] = Config{dockerLibraryRegistry, "busybox", "1.31"}
+	configs[BusyBoxV4] = Config{dockerLibraryRegistry, "busybox", "1.32"}
+	configs[FailedImage] = Config{dockerLibraryRegistry, "failed", "never-started"}
+	return configs
+}
+
+// GetImageConfigs returns the map of imageConfigs
+func GetImageConfigs() map[int]Config {
+	return imageConfigs
+}
+
+// GetConfig returns the Config object for an image
+func GetConfig(image int) Config {
+	return imageConfigs[image]
+}
+
+// GetE2EImage returns the fully qualified URI to an image (including version)
+func GetE2EImage(image int) string {
+	return fmt.Sprintf("%s/%s:%s", imageConfigs[image].registry, imageConfigs[image].name, imageConfigs[image].version)
+}
+
+// GetE2EImage returns the fully qualified URI to an image (including version)
+func (i *Config) GetE2EImage() string {
+	return fmt.Sprintf("%s/%s:%s", i.registry, i.name, i.version)
+}
+
+// GetPauseImageName returns the pause image name with proper version
+func GetPauseImageName() string {
+	return GetE2EImage(Pause)
+}
+
+// ReplaceRegistryInImageURL replaces the registry in the image URL with a custom one
+func ReplaceRegistryInImageURL(imageURL string) (string, error) {
+	parts := strings.Split(imageURL, "/")
+	countParts := len(parts)
+	registryAndUser := strings.Join(parts[:countParts-1], "/")
+
+	switch registryAndUser {
+	case "gcr.io/kubernetes-e2e-test-images":
+		registryAndUser = e2eRegistry
+	case "gcr.io/kubernetes-e2e-test-images/volume":
+		registryAndUser = e2eVolumeRegistry
+	case "k8s.gcr.io":
+		registryAndUser = gcRegistry
+	case "k8s.gcr.io/sig-storage":
+		registryAndUser = sigStorageRegistry
+	case "gcr.io/k8s-authenticated-test":
+		registryAndUser = PrivateRegistry
+	case "gcr.io/google-samples":
+		registryAndUser = sampleRegistry
+	case "gcr.io/gke-release":
+		registryAndUser = gcrReleaseRegistry
+	case "docker.io/library":
+		registryAndUser = dockerLibraryRegistry
+	default:
+		if countParts == 1 {
+			// We assume we found an image from docker hub library
+			// e.g. openjdk -> docker.io/library/openjdk
+			registryAndUser = dockerLibraryRegistry
+			break
+		}
+
+		return "", fmt.Errorf("Registry: %s is missing in test/utils/image/manifest.go, please add the registry, otherwise the test will fail on air-gapped clusters", registryAndUser)
+	}
+
+	return fmt.Sprintf("%s/%s", registryAndUser, parts[countParts-1]), nil
+}

vendor/github.com/gogo/protobuf/proto/text_parser.go

@@ -318,7 +318,7 @@ func unescape(s string) (ch string, tail string, err error) {
 		if i > utf8.MaxRune {
 			return "", "", fmt.Errorf(`\%c%s is not a valid Unicode code point`, r, ss)
 		}
-		return string(i), s, nil
+		return string(rune(i)), s, nil
 	}
 	return "", "", fmt.Errorf(`unknown escape \%c`, r)
 }

vendor/github.com/golang/protobuf/proto/registry.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"sync"
 
+	"google.golang.org/protobuf/reflect/protodesc"
 	"google.golang.org/protobuf/reflect/protoreflect"
 	"google.golang.org/protobuf/reflect/protoregistry"
 	"google.golang.org/protobuf/runtime/protoimpl"
@@ -62,14 +63,7 @@ func FileDescriptor(s filePath) fileDescGZIP {
 	// Find the descriptor in the v2 registry.
 	var b []byte
 	if fd, _ := protoregistry.GlobalFiles.FindFileByPath(s); fd != nil {
-		if fd, ok := fd.(interface{ ProtoLegacyRawDesc() []byte }); ok {
-			b = fd.ProtoLegacyRawDesc()
-		} else {
-			// TODO: Use protodesc.ToFileDescriptorProto to construct
-			// a descriptorpb.FileDescriptorProto and marshal it.
-			// However, doing so causes the proto package to have a dependency
-			// on descriptorpb, leading to cyclic dependency issues.
-		}
+		b, _ = Marshal(protodesc.ToFileDescriptorProto(fd))
 	}
 
 	// Locally cache the raw descriptor form for the file.

vendor/github.com/golang/protobuf/ptypes/any.go

@@ -19,6 +19,8 @@ const urlPrefix = "type.googleapis.com/"
 
 // AnyMessageName returns the message name contained in an anypb.Any message.
 // Most type assertions should use the Is function instead.
+//
+// Deprecated: Call the any.MessageName method instead.
 func AnyMessageName(any *anypb.Any) (string, error) {
 	name, err := anyMessageName(any)
 	return string(name), err
@@ -38,6 +40,8 @@ func anyMessageName(any *anypb.Any) (protoreflect.FullName, error) {
 }
 
 // MarshalAny marshals the given message m into an anypb.Any message.
+//
+// Deprecated: Call the anypb.New function instead.
 func MarshalAny(m proto.Message) (*anypb.Any, error) {
 	switch dm := m.(type) {
 	case DynamicAny:
@@ -58,6 +62,9 @@ func MarshalAny(m proto.Message) (*anypb.Any, error) {
 // Empty returns a new message of the type specified in an anypb.Any message.
 // It returns protoregistry.NotFound if the corresponding message type could not
 // be resolved in the global registry.
+//
+// Deprecated: Use protoregistry.GlobalTypes.FindMessageByName instead
+// to resolve the message name and create a new instance of it.
 func Empty(any *anypb.Any) (proto.Message, error) {
 	name, err := anyMessageName(any)
 	if err != nil {
@@ -76,6 +83,8 @@ func Empty(any *anypb.Any) (proto.Message, error) {
 //
 // The target message m may be a *DynamicAny message. If the underlying message
 // type could not be resolved, then this returns protoregistry.NotFound.
+//
+// Deprecated: Call the any.UnmarshalTo method instead.
 func UnmarshalAny(any *anypb.Any, m proto.Message) error {
 	if dm, ok := m.(*DynamicAny); ok {
 		if dm.Message == nil {
@@ -100,6 +109,8 @@ func UnmarshalAny(any *anypb.Any, m proto.Message) error {
 }
 
 // Is reports whether the Any message contains a message of the specified type.
+//
+// Deprecated: Call the any.MessageIs method instead.
 func Is(any *anypb.Any, m proto.Message) bool {
 	if any == nil || m == nil {
 		return false
@@ -119,6 +130,9 @@ func Is(any *anypb.Any, m proto.Message) bool {
 //   var x ptypes.DynamicAny
 //   if err := ptypes.UnmarshalAny(a, &x); err != nil { ... }
 //   fmt.Printf("unmarshaled message: %v", x.Message)
+//
+// Deprecated: Use the any.UnmarshalNew method instead to unmarshal
+// the any message contents into a new instance of the underlying message.
 type DynamicAny struct{ proto.Message }
 
 func (m DynamicAny) String() string {

vendor/github.com/golang/protobuf/ptypes/doc.go

@@ -3,4 +3,8 @@
 // license that can be found in the LICENSE file.
 
 // Package ptypes provides functionality for interacting with well-known types.
+//
+// Deprecated: Well-known types have specialized functionality directly
+// injected into the generated packages for each message type.
+// See the deprecation notice for each function for the suggested alternative.
 package ptypes

vendor/github.com/golang/protobuf/ptypes/duration.go

@@ -21,6 +21,8 @@ const (
 
 // Duration converts a durationpb.Duration to a time.Duration.
 // Duration returns an error if dur is invalid or overflows a time.Duration.
+//
+// Deprecated: Call the dur.AsDuration and dur.CheckValid methods instead.
 func Duration(dur *durationpb.Duration) (time.Duration, error) {
 	if err := validateDuration(dur); err != nil {
 		return 0, err
@@ -39,6 +41,8 @@ func Duration(dur *durationpb.Duration) (time.Duration, error) {
 }
 
 // DurationProto converts a time.Duration to a durationpb.Duration.
+//
+// Deprecated: Call the durationpb.New function instead.
 func DurationProto(d time.Duration) *durationpb.Duration {
 	nanos := d.Nanoseconds()
 	secs := nanos / 1e9

vendor/github.com/golang/protobuf/ptypes/timestamp.go

@@ -33,6 +33,8 @@ const (
 //
 // A nil Timestamp returns an error. The first return value in that case is
 // undefined.
+//
+// Deprecated: Call the ts.AsTime and ts.CheckValid methods instead.
 func Timestamp(ts *timestamppb.Timestamp) (time.Time, error) {
 	// Don't return the zero value on error, because corresponds to a valid
 	// timestamp. Instead return whatever time.Unix gives us.
@@ -46,6 +48,8 @@ func Timestamp(ts *timestamppb.Timestamp) (time.Time, error) {
 }
 
 // TimestampNow returns a google.protobuf.Timestamp for the current time.
+//
+// Deprecated: Call the timestamppb.Now function instead.
 func TimestampNow() *timestamppb.Timestamp {
 	ts, err := TimestampProto(time.Now())
 	if err != nil {
@@ -56,6 +60,8 @@ func TimestampNow() *timestamppb.Timestamp {
 
 // TimestampProto converts the time.Time to a google.protobuf.Timestamp proto.
 // It returns an error if the resulting Timestamp is invalid.
+//
+// Deprecated: Call the timestamppb.New function instead.
 func TimestampProto(t time.Time) (*timestamppb.Timestamp, error) {
 	ts := &timestamppb.Timestamp{
 		Seconds: t.Unix(),
@@ -69,6 +75,9 @@ func TimestampProto(t time.Time) (*timestamppb.Timestamp, error) {
 
 // TimestampString returns the RFC 3339 string for valid Timestamps.
 // For invalid Timestamps, it returns an error message in parentheses.
+//
+// Deprecated: Call the ts.AsTime method instead,
+// followed by a call to the Format method on the time.Time value.
 func TimestampString(ts *timestamppb.Timestamp) string {
 	t, err := Timestamp(ts)
 	if err != nil {

vendor/github.com/google/go-cmp/cmp/compare.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // Package cmp determines equality of values.
 //
@@ -100,8 +100,8 @@ func Equal(x, y interface{}, opts ...Option) bool {
 // same input values and options.
 //
 // The output is displayed as a literal in pseudo-Go syntax.
-// At the start of each line, a "-" prefix indicates an element removed from y,
-// a "+" prefix to indicates an element added to y, and the lack of a prefix
+// At the start of each line, a "-" prefix indicates an element removed from x,
+// a "+" prefix to indicates an element added from y, and the lack of a prefix
 // indicates an element common to both x and y. If possible, the output
 // uses fmt.Stringer.String or error.Error methods to produce more humanly
 // readable outputs. In such cases, the string is prefixed with either an

vendor/github.com/google/go-cmp/cmp/export_panic.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // +build purego
 

vendor/github.com/google/go-cmp/cmp/export_unsafe.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // +build !purego
 

vendor/github.com/google/go-cmp/cmp/internal/diff/debug_disable.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // +build !cmp_debug
 

vendor/github.com/google/go-cmp/cmp/internal/diff/debug_enable.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // +build cmp_debug
 

vendor/github.com/google/go-cmp/cmp/internal/diff/diff.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // Package diff implements an algorithm for producing edit-scripts.
 // The edit-script is a sequence of operations needed to transform one list
@@ -119,7 +119,7 @@ func (r Result) Similar() bool {
 	return r.NumSame+1 >= r.NumDiff
 }
 
-var randInt = rand.New(rand.NewSource(time.Now().Unix())).Intn(2)
+var randBool = rand.New(rand.NewSource(time.Now().Unix())).Intn(2) == 0
 
 // Difference reports whether two lists of lengths nx and ny are equal
 // given the definition of equality provided as f.
@@ -168,17 +168,6 @@ func Difference(nx, ny int, f EqualFunc) (es EditScript) {
 	// A vertical edge is equivalent to inserting a symbol from list Y.
 	// A diagonal edge is equivalent to a matching symbol between both X and Y.
 
-	// To ensure flexibility in changing the algorithm in the future,
-	// introduce some degree of deliberate instability.
-	// This is achieved by fiddling the zigzag iterator to start searching
-	// the graph starting from the bottom-right versus than the top-left.
-	// The result may differ depending on the starting search location,
-	// but still produces a valid edit script.
-	zigzagInit := randInt // either 0 or 1
-	if flags.Deterministic {
-		zigzagInit = 0
-	}
-
 	// Invariants:
 	//	• 0 ≤ fwdPath.X ≤ (fwdFrontier.X, revFrontier.X) ≤ revPath.X ≤ nx
 	//	• 0 ≤ fwdPath.Y ≤ (fwdFrontier.Y, revFrontier.Y) ≤ revPath.Y ≤ ny
@@ -197,6 +186,11 @@ func Difference(nx, ny int, f EqualFunc) (es EditScript) {
 	// approximately the square-root of the search budget.
 	searchBudget := 4 * (nx + ny) // O(n)
 
+	// Running the tests with the "cmp_debug" build tag prints a visualization
+	// of the algorithm running in real-time. This is educational for
+	// understanding how the algorithm works. See debug_enable.go.
+	f = debug.Begin(nx, ny, f, &fwdPath.es, &revPath.es)
+
 	// The algorithm below is a greedy, meet-in-the-middle algorithm for
 	// computing sub-optimal edit-scripts between two lists.
 	//
@@ -214,22 +208,28 @@ func Difference(nx, ny int, f EqualFunc) (es EditScript) {
 	//	frontier towards the opposite corner.
 	//	• This algorithm terminates when either the X coordinates or the
 	//	Y coordinates of the forward and reverse frontier points ever intersect.
-	//
+
 	// This algorithm is correct even if searching only in the forward direction
 	// or in the reverse direction. We do both because it is commonly observed
 	// that two lists commonly differ because elements were added to the front
 	// or end of the other list.
 	//
-	// Running the tests with the "cmp_debug" build tag prints a visualization
-	// of the algorithm running in real-time. This is educational for
-	// understanding how the algorithm works. See debug_enable.go.
-	f = debug.Begin(nx, ny, f, &fwdPath.es, &revPath.es)
-	for {
+	// Non-deterministically start with either the forward or reverse direction
+	// to introduce some deliberate instability so that we have the flexibility
+	// to change this algorithm in the future.
+	if flags.Deterministic || randBool {
+		goto forwardSearch
+	} else {
+		goto reverseSearch
+	}
+
+forwardSearch:
+	{
 		// Forward search from the beginning.
 		if fwdFrontier.X >= revFrontier.X || fwdFrontier.Y >= revFrontier.Y || searchBudget == 0 {
-			break
+			goto finishSearch
 		}
-		for stop1, stop2, i := false, false, zigzagInit; !(stop1 && stop2) && searchBudget > 0; i++ {
+		for stop1, stop2, i := false, false, 0; !(stop1 && stop2) && searchBudget > 0; i++ {
 			// Search in a diagonal pattern for a match.
 			z := zigzag(i)
 			p := point{fwdFrontier.X + z, fwdFrontier.Y - z}
@@ -262,10 +262,14 @@ func Difference(nx, ny int, f EqualFunc) (es EditScript) {
 		} else {
 			fwdFrontier.Y++
 		}
+		goto reverseSearch
+	}
 
+reverseSearch:
+	{
 		// Reverse search from the end.
 		if fwdFrontier.X >= revFrontier.X || fwdFrontier.Y >= revFrontier.Y || searchBudget == 0 {
-			break
+			goto finishSearch
 		}
 		for stop1, stop2, i := false, false, 0; !(stop1 && stop2) && searchBudget > 0; i++ {
 			// Search in a diagonal pattern for a match.
@@ -300,8 +304,10 @@ func Difference(nx, ny int, f EqualFunc) (es EditScript) {
 		} else {
 			revFrontier.Y--
 		}
+		goto forwardSearch
 	}
 
+finishSearch:
 	// Join the forward and reverse paths and then append the reverse path.
 	fwdPath.connect(revPath.point, f)
 	for i := len(revPath.es) - 1; i >= 0; i-- {

vendor/github.com/google/go-cmp/cmp/internal/flags/flags.go

@@ -1,6 +1,6 @@
 // Copyright 2019, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package flags
 

vendor/github.com/google/go-cmp/cmp/internal/flags/toolchain_legacy.go

@@ -1,6 +1,6 @@
 // Copyright 2019, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // +build !go1.10
 

vendor/github.com/google/go-cmp/cmp/internal/flags/toolchain_recent.go

@@ -1,6 +1,6 @@
 // Copyright 2019, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // +build go1.10
 

vendor/github.com/google/go-cmp/cmp/internal/function/func.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // Package function provides functionality for identifying function types.
 package function

vendor/github.com/google/go-cmp/cmp/internal/value/name.go

@@ -1,6 +1,6 @@
 // Copyright 2020, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package value
 

vendor/github.com/google/go-cmp/cmp/internal/value/pointer_purego.go

@@ -1,6 +1,6 @@
 // Copyright 2018, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // +build purego
 

vendor/github.com/google/go-cmp/cmp/internal/value/pointer_unsafe.go

@@ -1,6 +1,6 @@
 // Copyright 2018, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 // +build !purego
 

vendor/github.com/google/go-cmp/cmp/internal/value/sort.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package value
 

vendor/github.com/google/go-cmp/cmp/internal/value/zero.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package value
 

vendor/github.com/google/go-cmp/cmp/options.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package cmp
 

vendor/github.com/google/go-cmp/cmp/path.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package cmp
 

vendor/github.com/google/go-cmp/cmp/report.go

@@ -1,6 +1,6 @@
 // Copyright 2017, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package cmp
 

vendor/github.com/google/go-cmp/cmp/report_compare.go

@@ -1,6 +1,6 @@
 // Copyright 2019, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package cmp
 
@@ -79,7 +79,7 @@ func (opts formatOptions) verbosity() uint {
 	}
 }
 
-const maxVerbosityPreset = 3
+const maxVerbosityPreset = 6
 
 // verbosityPreset modifies the verbosity settings given an index
 // between 0 and maxVerbosityPreset, inclusive.
@@ -100,7 +100,7 @@ func verbosityPreset(opts formatOptions, i int) formatOptions {
 func (opts formatOptions) FormatDiff(v *valueNode, ptrs *pointerReferences) (out textNode) {
 	if opts.DiffMode == diffIdentical {
 		opts = opts.WithVerbosity(1)
-	} else {
+	} else if opts.verbosity() < 3 {
 		opts = opts.WithVerbosity(3)
 	}
 

vendor/github.com/google/go-cmp/cmp/report_references.go

@@ -1,6 +1,6 @@
 // Copyright 2020, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package cmp
 

vendor/github.com/google/go-cmp/cmp/report_reflect.go

@@ -1,6 +1,6 @@
 // Copyright 2019, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package cmp
 
@@ -351,6 +351,8 @@ func formatMapKey(v reflect.Value, disambiguate bool, ptrs *pointerReferences) s
 	opts.PrintAddresses = disambiguate
 	opts.AvoidStringer = disambiguate
 	opts.QualifiedNames = disambiguate
+	opts.VerbosityLevel = maxVerbosityPreset
+	opts.LimitVerbosity = true
 	s := opts.FormatValue(v, reflect.Map, ptrs).String()
 	return strings.TrimSpace(s)
 }

vendor/github.com/google/go-cmp/cmp/report_slices.go

@@ -1,6 +1,6 @@
 // Copyright 2019, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package cmp
 
@@ -26,8 +26,6 @@ func (opts formatOptions) CanFormatDiffSlice(v *valueNode) bool {
 		return false // No differences detected
 	case !v.ValueX.IsValid() || !v.ValueY.IsValid():
 		return false // Both values must be valid
-	case v.Type.Kind() == reflect.Slice && (v.ValueX.Len() == 0 || v.ValueY.Len() == 0):
-		return false // Both slice values have to be non-empty
 	case v.NumIgnored > 0:
 		return false // Some ignore option was used
 	case v.NumTransformed > 0:
@@ -45,7 +43,16 @@ func (opts formatOptions) CanFormatDiffSlice(v *valueNode) bool {
 		return false
 	}
 
-	switch t := v.Type; t.Kind() {
+	// Check whether this is an interface with the same concrete types.
+	t := v.Type
+	vx, vy := v.ValueX, v.ValueY
+	if t.Kind() == reflect.Interface && !vx.IsNil() && !vy.IsNil() && vx.Elem().Type() == vy.Elem().Type() {
+		vx, vy = vx.Elem(), vy.Elem()
+		t = vx.Type()
+	}
+
+	// Check whether we provide specialized diffing for this type.
+	switch t.Kind() {
 	case reflect.String:
 	case reflect.Array, reflect.Slice:
 		// Only slices of primitive types have specialized handling.
@@ -57,6 +64,11 @@ func (opts formatOptions) CanFormatDiffSlice(v *valueNode) bool {
 			return false
 		}
 
+		// Both slice values have to be non-empty.
+		if t.Kind() == reflect.Slice && (vx.Len() == 0 || vy.Len() == 0) {
+			return false
+		}
+
 		// If a sufficient number of elements already differ,
 		// use specialized formatting even if length requirement is not met.
 		if v.NumDiff > v.NumSame {
@@ -68,7 +80,7 @@ func (opts formatOptions) CanFormatDiffSlice(v *valueNode) bool {
 
 	// Use specialized string diffing for longer slices or strings.
 	const minLength = 64
-	return v.ValueX.Len() >= minLength && v.ValueY.Len() >= minLength
+	return vx.Len() >= minLength && vy.Len() >= minLength
 }
 
 // FormatDiffSlice prints a diff for the slices (or strings) represented by v.
@@ -77,6 +89,11 @@ func (opts formatOptions) CanFormatDiffSlice(v *valueNode) bool {
 func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
 	assert(opts.DiffMode == diffUnknown)
 	t, vx, vy := v.Type, v.ValueX, v.ValueY
+	if t.Kind() == reflect.Interface {
+		vx, vy = vx.Elem(), vy.Elem()
+		t = vx.Type()
+		opts = opts.WithTypeMode(emitType)
+	}
 
 	// Auto-detect the type of the data.
 	var isLinedText, isText, isBinary bool

vendor/github.com/google/go-cmp/cmp/report_text.go

@@ -1,6 +1,6 @@
 // Copyright 2019, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package cmp
 

vendor/github.com/google/go-cmp/cmp/report_value.go

@@ -1,6 +1,6 @@
 // Copyright 2019, The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE.md file.
+// license that can be found in the LICENSE file.
 
 package cmp
 

vendor/github.com/nxadm/tail/.gitignore

@@ -0,0 +1,3 @@
+.idea/
+.test/
+examples/_*

vendor/github.com/nxadm/tail/CHANGES.md

@@ -0,0 +1,56 @@
+# Version v1.4.7-v1.4.8
+* Documentation updates.
+* Small linter cleanups.
+* Added example in test. 
+
+# Version v1.4.6
+
+* Document the usage of Cleanup when re-reading a file (thanks to @lesovsky) for issue #18.
+* Add example directories with example and tests for issues.
+
+# Version v1.4.4-v1.4.5
+
+* Fix of checksum problem because of forced tag. No changes to the code.
+
+# Version v1.4.1
+
+* Incorporated PR 162 by by Mohammed902: "Simplify non-Windows build tag".
+
+# Version v1.4.0
+
+* Incorporated PR 9 by mschneider82: "Added seekinfo to Tail".
+
+# Version v1.3.1
+
+* Incorporated PR 7: "Fix deadlock when stopping on non-empty file/buffer",
+fixes upstream issue 93.
+
+
+# Version v1.3.0
+
+* Incorporated changes of unmerged upstream PR 149 by mezzi: "added line num
+to Line struct".
+
+# Version v1.2.1
+
+* Incorporated changes of unmerged upstream PR 128 by jadekler: "Compile-able
+code in readme".
+* Incorporated changes of unmerged upstream PR 130 by fgeller: "small change
+to comment wording".
+* Incorporated changes of unmerged upstream PR 133 by sm3142: "removed
+spurious newlines from log messages".
+
+# Version v1.2.0
+
+* Incorporated changes of unmerged upstream PR 126 by Code-Hex: "Solved the
+ problem for never return the last line if it's not followed by a newline".
+* Incorporated changes of unmerged upstream PR 131 by StoicPerlman: "Remove
+deprecated os.SEEK consts". The changes bumped the minimal supported Go
+release to 1.9.
+
+# Version v1.1.0
+
+* migration to go modules.
+* release of master branch of the dormant upstream, because it contains
+fixes and improvement no present in the tagged release.
+

vendor/github.com/nxadm/tail/Dockerfile

@@ -0,0 +1,19 @@
+FROM golang
+
+RUN mkdir -p $GOPATH/src/github.com/nxadm/tail/
+ADD . $GOPATH/src/github.com/nxadm/tail/
+
+# expecting to fetch dependencies successfully.
+RUN go get -v github.com/nxadm/tail
+
+# expecting to run the test successfully.
+RUN go test -v github.com/nxadm/tail
+
+# expecting to install successfully
+RUN go install -v github.com/nxadm/tail
+RUN go install -v github.com/nxadm/tail/cmd/gotail
+
+RUN $GOPATH/bin/gotail -h || true
+
+ENV PATH $GOPATH/bin:$PATH
+CMD ["gotail"]

vendor/github.com/nxadm/tail/LICENSE

@@ -0,0 +1,21 @@
+# The MIT License (MIT)
+
+# © Copyright 2015 Hewlett Packard Enterprise Development LP
+Copyright (c) 2014 ActiveState
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/nxadm/tail/README.md

@@ -0,0 +1,44 @@
+![ci](https://github.com/nxadm/tail/workflows/ci/badge.svg)[![Go Reference](https://pkg.go.dev/badge/github.com/nxadm/tail.svg)](https://pkg.go.dev/github.com/nxadm/tail)
+
+# tail functionality in Go
+
+nxadm/tail provides a Go library that emulates the features of the BSD `tail`
+program. The library comes with full support for truncation/move detection as
+it is designed to work with log rotation tools. The library works on all
+operating systems supported by Go, including POSIX systems like Linux and
+*BSD, and MS Windows. Go 1.9 is the oldest compiler release supported.
+
+A simple example:
+
+```Go
+// Create a tail
+t, err := tail.TailFile(
+	"/var/log/nginx.log", tail.Config{Follow: true, ReOpen: true})
+if err != nil {
+    panic(err)
+}
+
+// Print the text of each received line
+for line := range t.Lines {
+    fmt.Println(line.Text)
+}
+```
+
+See [API documentation](https://pkg.go.dev/github.com/nxadm/tail).
+
+## Installing
+
+    go get github.com/nxadm/tail/...
+
+## History
+
+This project is an active, drop-in replacement for the
+[abandoned](https://en.wikipedia.org/wiki/HPE_Helion) Go tail library at
+[hpcloud](https://github.com/hpcloud/tail). Next to
+[addressing open issues/PRs of the original project](https://github.com/nxadm/tail/issues/6),
+nxadm/tail continues the development by keeping up to date with the Go toolchain
+(e.g. go modules) and dependencies, completing the documentation, adding features
+and fixing bugs.
+
+## Examples
+Examples, e.g. used to debug an issue, are kept in the [examples directory](/examples).

vendor/github.com/nxadm/tail/go.mod

@@ -0,0 +1,8 @@
+module github.com/nxadm/tail
+
+go 1.13
+
+require (
+	github.com/fsnotify/fsnotify v1.4.9
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
+)

vendor/github.com/nxadm/tail/go.sum

@@ -0,0 +1,6 @@
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 h1:L2auWcuQIvxz9xSEqzESnV/QN/gNRXNApHi3fYwl2w0=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

vendor/github.com/nxadm/tail/ratelimiter/Licence

@@ -0,0 +1,7 @@
+Copyright (C) 2013 99designs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/nxadm/tail/ratelimiter/leakybucket.go

@@ -0,0 +1,97 @@
+// Package ratelimiter implements the Leaky Bucket ratelimiting algorithm with memcached and in-memory backends.
+package ratelimiter
+
+import (
+	"time"
+)
+
+type LeakyBucket struct {
+	Size         uint16
+	Fill         float64
+	LeakInterval time.Duration // time.Duration for 1 unit of size to leak
+	Lastupdate   time.Time
+	Now          func() time.Time
+}
+
+func NewLeakyBucket(size uint16, leakInterval time.Duration) *LeakyBucket {
+	bucket := LeakyBucket{
+		Size:         size,
+		Fill:         0,
+		LeakInterval: leakInterval,
+		Now:          time.Now,
+		Lastupdate:   time.Now(),
+	}
+
+	return &bucket
+}
+
+func (b *LeakyBucket) updateFill() {
+	now := b.Now()
+	if b.Fill > 0 {
+		elapsed := now.Sub(b.Lastupdate)
+
+		b.Fill -= float64(elapsed) / float64(b.LeakInterval)
+		if b.Fill < 0 {
+			b.Fill = 0
+		}
+	}
+	b.Lastupdate = now
+}
+
+func (b *LeakyBucket) Pour(amount uint16) bool {
+	b.updateFill()
+
+	var newfill float64 = b.Fill + float64(amount)
+
+	if newfill > float64(b.Size) {
+		return false
+	}
+
+	b.Fill = newfill
+
+	return true
+}
+
+// The time at which this bucket will be completely drained
+func (b *LeakyBucket) DrainedAt() time.Time {
+	return b.Lastupdate.Add(time.Duration(b.Fill * float64(b.LeakInterval)))
+}
+
+// The duration until this bucket is completely drained
+func (b *LeakyBucket) TimeToDrain() time.Duration {
+	return b.DrainedAt().Sub(b.Now())
+}
+
+func (b *LeakyBucket) TimeSinceLastUpdate() time.Duration {
+	return b.Now().Sub(b.Lastupdate)
+}
+
+type LeakyBucketSer struct {
+	Size         uint16
+	Fill         float64
+	LeakInterval time.Duration // time.Duration for 1 unit of size to leak
+	Lastupdate   time.Time
+}
+
+func (b *LeakyBucket) Serialise() *LeakyBucketSer {
+	bucket := LeakyBucketSer{
+		Size:         b.Size,
+		Fill:         b.Fill,
+		LeakInterval: b.LeakInterval,
+		Lastupdate:   b.Lastupdate,
+	}
+
+	return &bucket
+}
+
+func (b *LeakyBucketSer) DeSerialise() *LeakyBucket {
+	bucket := LeakyBucket{
+		Size:         b.Size,
+		Fill:         b.Fill,
+		LeakInterval: b.LeakInterval,
+		Lastupdate:   b.Lastupdate,
+		Now:          time.Now,
+	}
+
+	return &bucket
+}

vendor/github.com/nxadm/tail/ratelimiter/memory.go

@@ -0,0 +1,60 @@
+package ratelimiter
+
+import (
+	"errors"
+	"time"
+)
+
+const (
+	GC_SIZE   int           = 100
+	GC_PERIOD time.Duration = 60 * time.Second
+)
+
+type Memory struct {
+	store           map[string]LeakyBucket
+	lastGCCollected time.Time
+}
+
+func NewMemory() *Memory {
+	m := new(Memory)
+	m.store = make(map[string]LeakyBucket)
+	m.lastGCCollected = time.Now()
+	return m
+}
+
+func (m *Memory) GetBucketFor(key string) (*LeakyBucket, error) {
+
+	bucket, ok := m.store[key]
+	if !ok {
+		return nil, errors.New("miss")
+	}
+
+	return &bucket, nil
+}
+
+func (m *Memory) SetBucketFor(key string, bucket LeakyBucket) error {
+
+	if len(m.store) > GC_SIZE {
+		m.GarbageCollect()
+	}
+
+	m.store[key] = bucket
+
+	return nil
+}
+
+func (m *Memory) GarbageCollect() {
+	now := time.Now()
+
+	// rate limit GC to once per minute
+	if now.Unix() >= m.lastGCCollected.Add(GC_PERIOD).Unix() {
+		for key, bucket := range m.store {
+			// if the bucket is drained, then GC
+			if bucket.DrainedAt().Unix() < now.Unix() {
+				delete(m.store, key)
+			}
+		}
+
+		m.lastGCCollected = now
+	}
+}

vendor/github.com/nxadm/tail/ratelimiter/storage.go

@@ -0,0 +1,6 @@
+package ratelimiter
+
+type Storage interface {
+	GetBucketFor(string) (*LeakyBucket, error)
+	SetBucketFor(string, LeakyBucket) error
+}

vendor/github.com/nxadm/tail/tail.go

@@ -0,0 +1,455 @@
+// Copyright (c) 2019 FOSS contributors of https://github.com/nxadm/tail
+// Copyright (c) 2015 HPE Software Inc. All rights reserved.
+// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.
+
+//nxadm/tail provides a Go library that emulates the features of the BSD `tail`
+//program. The library comes with full support for truncation/move detection as
+//it is designed to work with log rotation tools. The library works on all
+//operating systems supported by Go, including POSIX systems like Linux and
+//*BSD, and MS Windows. Go 1.9 is the oldest compiler release supported.
+package tail
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/nxadm/tail/ratelimiter"
+	"github.com/nxadm/tail/util"
+	"github.com/nxadm/tail/watch"
+	"gopkg.in/tomb.v1"
+)
+
+var (
+	// ErrStop is returned when the tail of a file has been marked to be stopped.
+	ErrStop = errors.New("tail should now stop")
+)
+
+type Line struct {
+	Text     string    // The contents of the file
+	Num      int       // The line number
+	SeekInfo SeekInfo  // SeekInfo
+	Time     time.Time // Present time
+	Err      error     // Error from tail
+}
+
+// Deprecated: this function is no longer used internally and it has little of no
+// use in the API. As such, it will be removed from the API in a future major
+// release.
+//
+// NewLine returns a * pointer to a Line struct.
+func NewLine(text string, lineNum int) *Line {
+	return &Line{text, lineNum, SeekInfo{}, time.Now(), nil}
+}
+
+// SeekInfo represents arguments to io.Seek. See: https://golang.org/pkg/io/#SectionReader.Seek
+type SeekInfo struct {
+	Offset int64
+	Whence int
+}
+
+type logger interface {
+	Fatal(v ...interface{})
+	Fatalf(format string, v ...interface{})
+	Fatalln(v ...interface{})
+	Panic(v ...interface{})
+	Panicf(format string, v ...interface{})
+	Panicln(v ...interface{})
+	Print(v ...interface{})
+	Printf(format string, v ...interface{})
+	Println(v ...interface{})
+}
+
+// Config is used to specify how a file must be tailed.
+type Config struct {
+	// File-specifc
+	Location  *SeekInfo // Tail from this location. If nil, start at the beginning of the file
+	ReOpen    bool      // Reopen recreated files (tail -F)
+	MustExist bool      // Fail early if the file does not exist
+	Poll      bool      // Poll for file changes instead of using the default inotify
+	Pipe      bool      // The file is a named pipe (mkfifo)
+
+	// Generic IO
+	Follow      bool // Continue looking for new lines (tail -f)
+	MaxLineSize int  // If non-zero, split longer lines into multiple lines
+
+	// Optionally, use a ratelimiter (e.g. created by the ratelimiter/NewLeakyBucket function)
+	RateLimiter *ratelimiter.LeakyBucket
+
+	// Optionally use a Logger. When nil, the Logger is set to tail.DefaultLogger.
+	// To disable logging, set it to tail.DiscardingLogger
+	Logger logger
+}
+
+type Tail struct {
+	Filename string     // The filename
+	Lines    chan *Line // A consumable channel of *Line
+	Config              // Tail.Configuration
+
+	file    *os.File
+	reader  *bufio.Reader
+	lineNum int
+
+	watcher watch.FileWatcher
+	changes *watch.FileChanges
+
+	tomb.Tomb // provides: Done, Kill, Dying
+
+	lk sync.Mutex
+}
+
+var (
+	// DefaultLogger logs to os.Stderr and it is used when Config.Logger == nil
+	DefaultLogger = log.New(os.Stderr, "", log.LstdFlags)
+	// DiscardingLogger can be used to disable logging output
+	DiscardingLogger = log.New(ioutil.Discard, "", 0)
+)
+
+// TailFile begins tailing the file. And returns a pointer to a Tail struct
+// and an error. An output stream is made available via the Tail.Lines
+// channel (e.g. to be looped and printed). To handle errors during tailing,
+// after finishing reading from the Lines channel, invoke the `Wait` or `Err`
+// method on the returned *Tail.
+func TailFile(filename string, config Config) (*Tail, error) {
+	if config.ReOpen && !config.Follow {
+		util.Fatal("cannot set ReOpen without Follow.")
+	}
+
+	t := &Tail{
+		Filename: filename,
+		Lines:    make(chan *Line),
+		Config:   config,
+	}
+
+	// when Logger was not specified in config, use default logger
+	if t.Logger == nil {
+		t.Logger = DefaultLogger
+	}
+
+	if t.Poll {
+		t.watcher = watch.NewPollingFileWatcher(filename)
+	} else {
+		t.watcher = watch.NewInotifyFileWatcher(filename)
+	}
+
+	if t.MustExist {
+		var err error
+		t.file, err = OpenFile(t.Filename)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	go t.tailFileSync()
+
+	return t, nil
+}
+
+// Tell returns the file's current position, like stdio's ftell() and an error.
+// Beware that this value may not be completely accurate because one line from
+// the chan(tail.Lines) may have been read already.
+func (tail *Tail) Tell() (offset int64, err error) {
+	if tail.file == nil {
+		return
+	}
+	offset, err = tail.file.Seek(0, io.SeekCurrent)
+	if err != nil {
+		return
+	}
+
+	tail.lk.Lock()
+	defer tail.lk.Unlock()
+	if tail.reader == nil {
+		return
+	}
+
+	offset -= int64(tail.reader.Buffered())
+	return
+}
+
+// Stop stops the tailing activity.
+func (tail *Tail) Stop() error {
+	tail.Kill(nil)
+	return tail.Wait()
+}
+
+// StopAtEOF stops tailing as soon as the end of the file is reached. The function
+// returns an error,
+func (tail *Tail) StopAtEOF() error {
+	tail.Kill(errStopAtEOF)
+	return tail.Wait()
+}
+
+var errStopAtEOF = errors.New("tail: stop at eof")
+
+func (tail *Tail) close() {
+	close(tail.Lines)
+	tail.closeFile()
+}
+
+func (tail *Tail) closeFile() {
+	if tail.file != nil {
+		tail.file.Close()
+		tail.file = nil
+	}
+}
+
+func (tail *Tail) reopen() error {
+	tail.closeFile()
+	tail.lineNum = 0
+	for {
+		var err error
+		tail.file, err = OpenFile(tail.Filename)
+		if err != nil {
+			if os.IsNotExist(err) {
+				tail.Logger.Printf("Waiting for %s to appear...", tail.Filename)
+				if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil {
+					if err == tomb.ErrDying {
+						return err
+					}
+					return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err)
+				}
+				continue
+			}
+			return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err)
+		}
+		break
+	}
+	return nil
+}
+
+func (tail *Tail) readLine() (string, error) {
+	tail.lk.Lock()
+	line, err := tail.reader.ReadString('\n')
+	tail.lk.Unlock()
+	if err != nil {
+		// Note ReadString "returns the data read before the error" in
+		// case of an error, including EOF, so we return it as is. The
+		// caller is expected to process it if err is EOF.
+		return line, err
+	}
+
+	line = strings.TrimRight(line, "\n")
+
+	return line, err
+}
+
+func (tail *Tail) tailFileSync() {
+	defer tail.Done()
+	defer tail.close()
+
+	if !tail.MustExist {
+		// deferred first open.
+		err := tail.reopen()
+		if err != nil {
+			if err != tomb.ErrDying {
+				tail.Kill(err)
+			}
+			return
+		}
+	}
+
+	// Seek to requested location on first open of the file.
+	if tail.Location != nil {
+		_, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence)
+		if err != nil {
+			tail.Killf("Seek error on %s: %s", tail.Filename, err)
+			return
+		}
+	}
+
+	tail.openReader()
+
+	// Read line by line.
+	for {
+		// do not seek in named pipes
+		if !tail.Pipe {
+			// grab the position in case we need to back up in the event of a half-line
+			if _, err := tail.Tell(); err != nil {
+				tail.Kill(err)
+				return
+			}
+		}
+
+		line, err := tail.readLine()
+
+		// Process `line` even if err is EOF.
+		if err == nil {
+			cooloff := !tail.sendLine(line)
+			if cooloff {
+				// Wait a second before seeking till the end of
+				// file when rate limit is reached.
+				msg := ("Too much log activity; waiting a second before resuming tailing")
+				offset, _ := tail.Tell()
+				tail.Lines <- &Line{msg, tail.lineNum, SeekInfo{Offset: offset}, time.Now(), errors.New(msg)}
+				select {
+				case <-time.After(time.Second):
+				case <-tail.Dying():
+					return
+				}
+				if err := tail.seekEnd(); err != nil {
+					tail.Kill(err)
+					return
+				}
+			}
+		} else if err == io.EOF {
+			if !tail.Follow {
+				if line != "" {
+					tail.sendLine(line)
+				}
+				return
+			}
+
+			if tail.Follow && line != "" {
+				tail.sendLine(line)
+				if err := tail.seekEnd(); err != nil {
+					tail.Kill(err)
+					return
+				}
+			}
+
+			// When EOF is reached, wait for more data to become
+			// available. Wait strategy is based on the `tail.watcher`
+			// implementation (inotify or polling).
+			err := tail.waitForChanges()
+			if err != nil {
+				if err != ErrStop {
+					tail.Kill(err)
+				}
+				return
+			}
+		} else {
+			// non-EOF error
+			tail.Killf("Error reading %s: %s", tail.Filename, err)
+			return
+		}
+
+		select {
+		case <-tail.Dying():
+			if tail.Err() == errStopAtEOF {
+				continue
+			}
+			return
+		default:
+		}
+	}
+}
+
+// waitForChanges waits until the file has been appended, deleted,
+// moved or truncated. When moved or deleted - the file will be
+// reopened if ReOpen is true. Truncated files are always reopened.
+func (tail *Tail) waitForChanges() error {
+	if tail.changes == nil {
+		pos, err := tail.file.Seek(0, io.SeekCurrent)
+		if err != nil {
+			return err
+		}
+		tail.changes, err = tail.watcher.ChangeEvents(&tail.Tomb, pos)
+		if err != nil {
+			return err
+		}
+	}
+
+	select {
+	case <-tail.changes.Modified:
+		return nil
+	case <-tail.changes.Deleted:
+		tail.changes = nil
+		if tail.ReOpen {
+			// XXX: we must not log from a library.
+			tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename)
+			if err := tail.reopen(); err != nil {
+				return err
+			}
+			tail.Logger.Printf("Successfully reopened %s", tail.Filename)
+			tail.openReader()
+			return nil
+		}
+		tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename)
+		return ErrStop
+	case <-tail.changes.Truncated:
+		// Always reopen truncated files (Follow is true)
+		tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename)
+		if err := tail.reopen(); err != nil {
+			return err
+		}
+		tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename)
+		tail.openReader()
+		return nil
+	case <-tail.Dying():
+		return ErrStop
+	}
+}
+
+func (tail *Tail) openReader() {
+	tail.lk.Lock()
+	if tail.MaxLineSize > 0 {
+		// add 2 to account for newline characters
+		tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2)
+	} else {
+		tail.reader = bufio.NewReader(tail.file)
+	}
+	tail.lk.Unlock()
+}
+
+func (tail *Tail) seekEnd() error {
+	return tail.seekTo(SeekInfo{Offset: 0, Whence: io.SeekEnd})
+}
+
+func (tail *Tail) seekTo(pos SeekInfo) error {
+	_, err := tail.file.Seek(pos.Offset, pos.Whence)
+	if err != nil {
+		return fmt.Errorf("Seek error on %s: %s", tail.Filename, err)
+	}
+	// Reset the read buffer whenever the file is re-seek'ed
+	tail.reader.Reset(tail.file)
+	return nil
+}
+
+// sendLine sends the line(s) to Lines channel, splitting longer lines
+// if necessary. Return false if rate limit is reached.
+func (tail *Tail) sendLine(line string) bool {
+	now := time.Now()
+	lines := []string{line}
+
+	// Split longer lines
+	if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize {
+		lines = util.PartitionString(line, tail.MaxLineSize)
+	}
+
+	for _, line := range lines {
+		tail.lineNum++
+		offset, _ := tail.Tell()
+		select {
+		case tail.Lines <- &Line{line, tail.lineNum, SeekInfo{Offset: offset}, now, nil}:
+		case <-tail.Dying():
+			return true
+		}
+	}
+
+	if tail.Config.RateLimiter != nil {
+		ok := tail.Config.RateLimiter.Pour(uint16(len(lines)))
+		if !ok {
+			tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.",
+				tail.Filename)
+			return false
+		}
+	}
+
+	return true
+}
+
+// Cleanup removes inotify watches added by the tail package. This function is
+// meant to be invoked from a process's exit handler. Linux kernel may not
+// automatically remove inotify watches after the process exits.
+// If you plan to re-read a file, don't call Cleanup in between.
+func (tail *Tail) Cleanup() {
+	watch.Cleanup(tail.Filename)
+}

vendor/github.com/nxadm/tail/tail_posix.go

@@ -0,0 +1,17 @@
+// Copyright (c) 2019 FOSS contributors of https://github.com/nxadm/tail
+// +build !windows
+
+package tail
+
+import (
+	"os"
+)
+
+// Deprecated: this function is only useful internally and, as such,
+// it will be removed from the API in a future major release.
+//
+// OpenFile proxies a os.Open call for a file so it can be correctly tailed
+// on POSIX and non-POSIX OSes like MS Windows.
+func OpenFile(name string) (file *os.File, err error) {
+	return os.Open(name)
+}

vendor/github.com/nxadm/tail/tail_windows.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2019 FOSS contributors of https://github.com/nxadm/tail
+// +build windows
+
+package tail
+
+import (
+	"os"
+
+	"github.com/nxadm/tail/winfile"
+)
+
+// Deprecated: this function is only useful internally and, as such,
+// it will be removed from the API in a future major release.
+//
+// OpenFile proxies a os.Open call for a file so it can be correctly tailed
+// on POSIX and non-POSIX OSes like MS Windows.
+func OpenFile(name string) (file *os.File, err error) {
+	return winfile.OpenFile(name, os.O_RDONLY, 0)
+}

vendor/github.com/nxadm/tail/util/util.go

@@ -0,0 +1,49 @@
+// Copyright (c) 2019 FOSS contributors of https://github.com/nxadm/tail
+// Copyright (c) 2015 HPE Software Inc. All rights reserved.
+// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.
+
+package util
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"runtime/debug"
+)
+
+type Logger struct {
+	*log.Logger
+}
+
+var LOGGER = &Logger{log.New(os.Stderr, "", log.LstdFlags)}
+
+// fatal is like panic except it displays only the current goroutine's stack.
+func Fatal(format string, v ...interface{}) {
+	// https://github.com/nxadm/log/blob/master/log.go#L45
+	LOGGER.Output(2, fmt.Sprintf("FATAL -- "+format, v...)+"\n"+string(debug.Stack()))
+	os.Exit(1)
+}
+
+// partitionString partitions the string into chunks of given size,
+// with the last chunk of variable size.
+func PartitionString(s string, chunkSize int) []string {
+	if chunkSize <= 0 {
+		panic("invalid chunkSize")
+	}
+	length := len(s)
+	chunks := 1 + length/chunkSize
+	start := 0
+	end := chunkSize
+	parts := make([]string, 0, chunks)
+	for {
+		if end > length {
+			end = length
+		}
+		parts = append(parts, s[start:end])
+		if end == length {
+			break
+		}
+		start, end = end, end+chunkSize
+	}
+	return parts
+}

vendor/github.com/nxadm/tail/watch/filechanges.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2019 FOSS contributors of https://github.com/nxadm/tail
+package watch
+
+type FileChanges struct {
+	Modified  chan bool // Channel to get notified of modifications
+	Truncated chan bool // Channel to get notified of truncations
+	Deleted   chan bool // Channel to get notified of deletions/renames
+}
+
+func NewFileChanges() *FileChanges {
+	return &FileChanges{
+		make(chan bool, 1), make(chan bool, 1), make(chan bool, 1)}
+}
+
+func (fc *FileChanges) NotifyModified() {
+	sendOnlyIfEmpty(fc.Modified)
+}
+
+func (fc *FileChanges) NotifyTruncated() {
+	sendOnlyIfEmpty(fc.Truncated)
+}
+
+func (fc *FileChanges) NotifyDeleted() {
+	sendOnlyIfEmpty(fc.Deleted)
+}
+
+// sendOnlyIfEmpty sends on a bool channel only if the channel has no
+// backlog to be read by other goroutines. This concurrency pattern
+// can be used to notify other goroutines if and only if they are
+// looking for it (i.e., subsequent notifications can be compressed
+// into one).
+func sendOnlyIfEmpty(ch chan bool) {
+	select {
+	case ch <- true:
+	default:
+	}
+}