From 3421d8fbd87afc849489da0a0850a7b00a28c2ff Mon Sep 17 00:00:00 2001 From: berg Date: Fri, 22 Jul 2022 18:47:37 +0800 Subject: [PATCH] add rollout history controller Signed-off-by: liheng.zms Signed-off-by: yike21 Signed-off-by: yike21 --- .../e2e-rollouthistory-cloneset-1.23.yaml | 110 + .../e2e-rollouthistory-statefulset-1.23.yaml | 110 + api/v1alpha1/rollouthistory_types.go | 205 ++ api/v1alpha1/zz_generated.deepcopy.go | 289 +- .../rollouts.kruise.io_rollouthistories.yaml | 2959 +++++++++++++++++ config/crd/kustomization.yaml | 5 +- config/rbac/role.yaml | 26 + main.go | 22 +- .../rollouthistory_controller.go | 100 + .../rollouthistory_event_handler.go | 669 ++++ pkg/controller/rollouthistory/utils.go | 39 + test/e2e/rollouthistory_test.go | 1894 +++++++++++ 12 files changed, 6420 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/e2e-rollouthistory-cloneset-1.23.yaml create mode 100644 .github/workflows/e2e-rollouthistory-statefulset-1.23.yaml create mode 100644 api/v1alpha1/rollouthistory_types.go create mode 100644 config/crd/bases/rollouts.kruise.io_rollouthistories.yaml create mode 100644 pkg/controller/rollouthistory/rollouthistory_controller.go create mode 100644 pkg/controller/rollouthistory/rollouthistory_event_handler.go create mode 100644 pkg/controller/rollouthistory/utils.go create mode 100644 test/e2e/rollouthistory_test.go diff --git a/.github/workflows/e2e-rollouthistory-cloneset-1.23.yaml b/.github/workflows/e2e-rollouthistory-cloneset-1.23.yaml new file mode 100644 index 0000000..99fcd08 --- /dev/null +++ b/.github/workflows/e2e-rollouthistory-cloneset-1.23.yaml @@ -0,0 +1,110 @@ +name: E2E-RolloutHistory-CloneSet-1.23 + +on: + push: + branches: + - master + - release-* + pull_request: {} + workflow_dispatch: {} + +env: + # Common versions + GO_VERSION: '1.17' + KIND_IMAGE: 'kindest/node:v1.23.3' + KIND_CLUSTER_NAME: 'ci-testing' + +jobs: + + rollout: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + node_image: ${{ env.KIND_IMAGE }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + config: ./test/kind-conf.yaml + - name: Build image + run: | + export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}" + docker build --pull --no-cache . -t $IMAGE + kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; } + - name: Install Kruise + run: | + set -ex + kubectl cluster-info + make helm + helm repo add openkruise https://openkruise.github.io/charts/ + helm repo update + helm install kruise openkruise/kruise + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + echo "Wait for kruise-manager ready successfully" + else + echo "Timeout to wait for kruise-manager ready" + exit 1 + fi + - name: Install Kruise Rollout + run: | + set -ex + kubectl cluster-info + IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + set -e + if [ "$PODS" -eq "1" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + kubectl get node -o yaml + kubectl get all -n kruise-rollout -o yaml + set -e + if [ "$PODS" -eq "1" ]; then + echo "Wait for kruise-rollout ready successfully" + else + echo "Timeout to wait for kruise-rollout ready" + exit 1 + fi + - name: Run E2E Tests + run: | + export KUBECONFIG=/home/runner/.kube/config + make ginkgo + set +e + ./bin/ginkgo -timeout 60m -v --focus='CloneSet canary rollout with RolloutHistory' test/e2e + retVal=$? + # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout + restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') + if [ "${restartCount}" -eq "0" ];then + echo "Kruise-rollout has not restarted" + else + kubectl get pod -n kruise-rollout --no-headers + echo "Kruise-rollout has restarted, abort!!!" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + exit 1 + fi + exit $retVal \ No newline at end of file diff --git a/.github/workflows/e2e-rollouthistory-statefulset-1.23.yaml b/.github/workflows/e2e-rollouthistory-statefulset-1.23.yaml new file mode 100644 index 0000000..1c9203a --- /dev/null +++ b/.github/workflows/e2e-rollouthistory-statefulset-1.23.yaml @@ -0,0 +1,110 @@ +name: E2E-RolloutHistory-StatefulSet-1.23 + +on: + push: + branches: + - master + - release-* + pull_request: {} + workflow_dispatch: {} + +env: + # Common versions + GO_VERSION: '1.17' + KIND_IMAGE: 'kindest/node:v1.23.3' + KIND_CLUSTER_NAME: 'ci-testing' + +jobs: + + rollout: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + node_image: ${{ env.KIND_IMAGE }} + cluster_name: ${{ env.KIND_CLUSTER_NAME }} + config: ./test/kind-conf.yaml + - name: Build image + run: | + export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}" + docker build --pull --no-cache . -t $IMAGE + kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; } + - name: Install Kruise + run: | + set -ex + kubectl cluster-info + make helm + helm repo add openkruise https://openkruise.github.io/charts/ + helm repo update + helm install kruise openkruise/kruise + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-system | grep '1/1' | grep kruise-controller-manager | wc -l) + set -e + if [ "$PODS" -eq "2" ]; then + echo "Wait for kruise-manager ready successfully" + else + echo "Timeout to wait for kruise-manager ready" + exit 1 + fi + - name: Install Kruise Rollout + run: | + set -ex + kubectl cluster-info + IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh + for ((i=1;i<10;i++)); + do + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + set -e + if [ "$PODS" -eq "1" ]; then + break + fi + sleep 3 + done + set +e + PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l) + kubectl get node -o yaml + kubectl get all -n kruise-rollout -o yaml + set -e + if [ "$PODS" -eq "1" ]; then + echo "Wait for kruise-rollout ready successfully" + else + echo "Timeout to wait for kruise-rollout ready" + exit 1 + fi + - name: Run E2E Tests + run: | + export KUBECONFIG=/home/runner/.kube/config + make ginkgo + set +e + ./bin/ginkgo -timeout 60m -v --focus='StatefulSet canary rollout with RolloutHistory' test/e2e + retVal=$? + # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout + restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}') + if [ "${restartCount}" -eq "0" ];then + echo "Kruise-rollout has not restarted" + else + kubectl get pod -n kruise-rollout --no-headers + echo "Kruise-rollout has restarted, abort!!!" + kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout + exit 1 + fi + exit $retVal \ No newline at end of file diff --git a/api/v1alpha1/rollouthistory_types.go b/api/v1alpha1/rollouthistory_types.go new file mode 100644 index 0000000..365f67a --- /dev/null +++ b/api/v1alpha1/rollouthistory_types.go @@ -0,0 +1,205 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// RolloutHistorySpec defines the desired state of RolloutHistory +type RolloutHistorySpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // RolloutWrapper indicates information of the rollout related with rollouthistory + RolloutWrapper RolloutWrapper `json:"rolloutWrapper,omitempty"` + + // Workload indicates information of the workload, such as cloneset, deployment, advanced statefulset + Workload Workload `json:"workload,omitempty"` + + // ServiceWrapper indicates information of the service related with workload + ServiceWrapper ServiceWrapper `json:"serviceWrapper,omitempty"` + + // TrafficRoutingWrapper indicates information of traffic route related with workload + TrafficRoutingWrapper TrafficRoutingWrapper `json:"trafficRoutingWrapper,omitempty"` +} + +// RolloutWrapper indicates information of the rollout related +type RolloutWrapper struct { + // Name indicates the rollout name + Name string `json:"name,omitempty"` + // Rollout indecates the related rollout + Rollout Rollout `json:"rollout,omitempty"` +} + +// ServiceWrapper indicates information of the service related +type ServiceWrapper struct { + // Name indicates the service name + Name string `json:"name,omitempty"` + // Service indicates the service + Service *v1.Service `json:"service,omitempty"` +} + +// TrafficRoutingWrapper indicates information of Gateway API or Ingress +type TrafficRoutingWrapper struct { + // Ingress indicates information of ingress + // +optional + IngressWrapper *IngressWrapper `json:"ingressWrapper,omitempty"` + // HTTPRouteWrapper indacates information of Gateway API + // +optional + HTTPRouteWrapper *HTTPRouteWrapper `json:"httpRouteWrapper,omitempty"` +} + +// IngressWrapper indicates information of the ingress related +type IngressWrapper struct { + // Name indicates the ingress name + Name string `json:"name,omitempty"` + // Ingress indicates the ingress + Ingress *networkingv1.Ingress `json:"ingress,omitempty"` +} + +// HTTPRouteWrapper indicates information of gateway API +type HTTPRouteWrapper struct { + //Name indicates the httproute name + Name string `json:"name,omitempty"` + // HTTPRoute indicates the HTTPRoute + HTTPRoute *v1alpha2.HTTPRoute `json:"httpRoute,omitempty"` +} + +// Workload indicates information of the workload, such as cloneset, deployment, advanced statefulset +type Workload struct { + metav1.TypeMeta `json:",inline"` + // Name indicates the workload name + Name string `json:"name,omitempty"` + // Label selector for pods. + // It must match the pod template's labels. + Selector *metav1.LabelSelector `json:"selector" protobuf:"bytes,2,opt,name=selector"` + // Number of desired pods. This is a pointer to distinguish between explicit + // zero and not specified. Defaults to 1. + // +optional + Replicas *int32 `json:"replicas,omitempty" protobuf:"varint,1,opt,name=replicas"` + // Type of deployment or CloneSetUpdateStrategy. Can be "Recreate" or "RollingUpdate". + // +optional + Type string `json:"type,omitempty" protobuf:"bytes,1,opt,name=type,casttype=DeploymentStrategyType"` + // Template describes the pods that will be created. + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Template v1.PodTemplateSpec `json:"template"` +} + +// RolloutHistoryStatus defines the observed state of RolloutHistory +type RolloutHistoryStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Phase indicates phase of RolloutHistory, such as "pending", "progressing", "completed" + Phase string `json:"phase,omitempty"` + // CanaryStepIndex indicates the current step + CanaryStepIndex *int32 `json:"canaryStepIndex,omitempty"` + // CanaryStepState indicates state of this rollout revision, such as "init", "pending", "update", "terminated", "completed", "cancelled", whick is upon rollout canary_step_state + CanaryStepState string `json:"canaryStepState,omitempty"` + // RolloutState indicates the rollouts status + RolloutState RolloutState `json:"rolloutState,omitempty"` + // canaryStepPods indicates the pods released + CanaryStepPods []CanaryStepPods `json:"canaryStepPods,omitempty"` +} + +// RolloutState indicates the rollouts status +type RolloutState struct { + // RolloutPhase is the rollout phase. + RolloutPhase RolloutPhase `json:"rolloutPhase,omitempty"` + // Message provides details on why the rollout is in its current phase + Message string `json:"message,omitempty"` +} + +// CanaryStepPods indicates the pods for a revision +type CanaryStepPods struct { + // StepIndex indicates the step index + StepIndex int32 `json:"stepIndex,omitempty"` + // Pods indicates the pods information + Pods []Pod `json:"pods,omitempty"` + // PodsInTotal indicates the num of new pods released by now + PodsInTotal int32 `json:"podsInTotal"` + // PodsInStep indicates the num of new pods released this step + PodsInStep int32 `json:"podsInStep"` +} + +// Pod indicates the information of a pod, including name, ip, node_name. +type Pod struct { + // Name indicates the node name + Name string `json:"name,omitempty"` + // IP indicates the pod ip + IP string `json:"ip,omitempty"` + // Node indicates the node which pod is located at + Node string `json:"node,omitempty"` +} + +// CanaryStepState indicates canary-phase of rollouthistory when user do a rollout +const ( + CanaryStateInit string = "init" + CanaryStatePending string = "pending" + CanaryStateUpdated string = "updated" + CanaryStateTerminated string = "terminated" + CanaryStateCompleted string = "completed" +) + +// Phase indicates rollouthistory status/phase +const ( + PhaseInit string = "init" + PhaseCompleted string = "completed" + PhaseProgressing string = "progressing" +) + +// MaxRolloutHistoryNum indicates how many rollouthistories there can be at most +const MaxRolloutHistoryNum int = 10 + +// +genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="PHASE",type="string",JSONPath=".status.phase",description="Phase indicates phase of RolloutHistory, such as pending, progressing, completed" +//+kubebuilder:printcolumn:name="CURRENT_STEP",type="integer",JSONPath=".status.canaryStepIndex",description="CanaryStepIndex indicates the current step" +//+kubebuilder:printcolumn:name="CURRENT_STATE",type="string",JSONPath=".status.canaryStepState",description="CanaryStepState indicates state of this rollout revision, such as init, pending, update, terminated, completed, cancelled, whick is upon rollout canary_step_state" +//+kubebuilder:printcolumn:name="MESSAGE",type="string",JSONPath=".status.rolloutState.message",description="Message provides details on why the rollout is in its current phase" +//+kubebuilder:printcolumn:name="AGE",type=date,JSONPath=".metadata.creationTimestamp" + +// RolloutHistory is the Schema for the rollouthistories API +type RolloutHistory struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RolloutHistorySpec `json:"spec,omitempty"` + Status RolloutHistoryStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// RolloutHistoryList contains a list of RolloutHistory +type RolloutHistoryList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []RolloutHistory `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RolloutHistory{}, &RolloutHistoryList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 39eb6a6..5407c6e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,4 +1,3 @@ -//go:build !ignore_autogenerated // +build !ignore_autogenerated /* @@ -22,8 +21,12 @@ limitations under the License. package v1alpha1 import ( + "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/gateway-api/apis/v1alpha2" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -199,6 +202,26 @@ func (in *CanaryStep) DeepCopy() *CanaryStep { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CanaryStepPods) DeepCopyInto(out *CanaryStepPods) { + *out = *in + if in.Pods != nil { + in, out := &in.Pods, &out.Pods + *out = make([]Pod, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryStepPods. +func (in *CanaryStepPods) DeepCopy() *CanaryStepPods { + if in == nil { + return nil + } + out := new(CanaryStepPods) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CanaryStrategy) DeepCopyInto(out *CanaryStrategy) { *out = *in @@ -252,6 +275,26 @@ func (in *GatewayTrafficRouting) DeepCopy() *GatewayTrafficRouting { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPRouteWrapper) DeepCopyInto(out *HTTPRouteWrapper) { + *out = *in + if in.HTTPRoute != nil { + in, out := &in.HTTPRoute, &out.HTTPRoute + *out = new(v1alpha2.HTTPRoute) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPRouteWrapper. +func (in *HTTPRouteWrapper) DeepCopy() *HTTPRouteWrapper { + if in == nil { + return nil + } + out := new(HTTPRouteWrapper) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressTrafficRouting) DeepCopyInto(out *IngressTrafficRouting) { *out = *in @@ -267,6 +310,26 @@ func (in *IngressTrafficRouting) DeepCopy() *IngressTrafficRouting { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IngressWrapper) DeepCopyInto(out *IngressWrapper) { + *out = *in + if in.Ingress != nil { + in, out := &in.Ingress, &out.Ingress + *out = new(networkingv1.Ingress) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressWrapper. +func (in *IngressWrapper) DeepCopy() *IngressWrapper { + if in == nil { + return nil + } + out := new(IngressWrapper) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectRef) DeepCopyInto(out *ObjectRef) { *out = *in @@ -287,6 +350,21 @@ func (in *ObjectRef) DeepCopy() *ObjectRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Pod) DeepCopyInto(out *Pod) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pod. +func (in *Pod) DeepCopy() *Pod { + if in == nil { + return nil + } + out := new(Pod) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReleaseBatch) DeepCopyInto(out *ReleaseBatch) { *out = *in @@ -372,6 +450,112 @@ func (in *RolloutCondition) DeepCopy() *RolloutCondition { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutHistory) DeepCopyInto(out *RolloutHistory) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutHistory. +func (in *RolloutHistory) DeepCopy() *RolloutHistory { + if in == nil { + return nil + } + out := new(RolloutHistory) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RolloutHistory) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutHistoryList) DeepCopyInto(out *RolloutHistoryList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RolloutHistory, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutHistoryList. +func (in *RolloutHistoryList) DeepCopy() *RolloutHistoryList { + if in == nil { + return nil + } + out := new(RolloutHistoryList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RolloutHistoryList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutHistorySpec) DeepCopyInto(out *RolloutHistorySpec) { + *out = *in + in.RolloutWrapper.DeepCopyInto(&out.RolloutWrapper) + in.Workload.DeepCopyInto(&out.Workload) + in.ServiceWrapper.DeepCopyInto(&out.ServiceWrapper) + in.TrafficRoutingWrapper.DeepCopyInto(&out.TrafficRoutingWrapper) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutHistorySpec. +func (in *RolloutHistorySpec) DeepCopy() *RolloutHistorySpec { + if in == nil { + return nil + } + out := new(RolloutHistorySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutHistoryStatus) DeepCopyInto(out *RolloutHistoryStatus) { + *out = *in + if in.CanaryStepIndex != nil { + in, out := &in.CanaryStepIndex, &out.CanaryStepIndex + *out = new(int32) + **out = **in + } + out.RolloutState = in.RolloutState + if in.CanaryStepPods != nil { + in, out := &in.CanaryStepPods, &out.CanaryStepPods + *out = make([]CanaryStepPods, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutHistoryStatus. +func (in *RolloutHistoryStatus) DeepCopy() *RolloutHistoryStatus { + if in == nil { + return nil + } + out := new(RolloutHistoryStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RolloutList) DeepCopyInto(out *RolloutList) { *out = *in @@ -441,6 +625,21 @@ func (in *RolloutSpec) DeepCopy() *RolloutSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutState) DeepCopyInto(out *RolloutState) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutState. +func (in *RolloutState) DeepCopy() *RolloutState { + if in == nil { + return nil + } + out := new(RolloutState) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RolloutStatus) DeepCopyInto(out *RolloutStatus) { *out = *in @@ -488,6 +687,42 @@ func (in *RolloutStrategy) DeepCopy() *RolloutStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutWrapper) DeepCopyInto(out *RolloutWrapper) { + *out = *in + in.Rollout.DeepCopyInto(&out.Rollout) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutWrapper. +func (in *RolloutWrapper) DeepCopy() *RolloutWrapper { + if in == nil { + return nil + } + out := new(RolloutWrapper) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceWrapper) DeepCopyInto(out *ServiceWrapper) { + *out = *in + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = new(v1.Service) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceWrapper. +func (in *ServiceWrapper) DeepCopy() *ServiceWrapper { + if in == nil { + return nil + } + out := new(ServiceWrapper) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TrafficRouting) DeepCopyInto(out *TrafficRouting) { *out = *in @@ -513,6 +748,58 @@ func (in *TrafficRouting) DeepCopy() *TrafficRouting { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TrafficRoutingWrapper) DeepCopyInto(out *TrafficRoutingWrapper) { + *out = *in + if in.IngressWrapper != nil { + in, out := &in.IngressWrapper, &out.IngressWrapper + *out = new(IngressWrapper) + (*in).DeepCopyInto(*out) + } + if in.HTTPRouteWrapper != nil { + in, out := &in.HTTPRouteWrapper, &out.HTTPRouteWrapper + *out = new(HTTPRouteWrapper) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRoutingWrapper. +func (in *TrafficRoutingWrapper) DeepCopy() *TrafficRoutingWrapper { + if in == nil { + return nil + } + out := new(TrafficRoutingWrapper) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Workload) DeepCopyInto(out *Workload) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Workload. +func (in *Workload) DeepCopy() *Workload { + if in == nil { + return nil + } + out := new(Workload) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadRef) DeepCopyInto(out *WorkloadRef) { *out = *in diff --git a/config/crd/bases/rollouts.kruise.io_rollouthistories.yaml b/config/crd/bases/rollouts.kruise.io_rollouthistories.yaml new file mode 100644 index 0000000..d674a7f --- /dev/null +++ b/config/crd/bases/rollouts.kruise.io_rollouthistories.yaml @@ -0,0 +1,2959 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: rollouthistories.rollouts.kruise.io +spec: + group: rollouts.kruise.io + names: + kind: RolloutHistory + listKind: RolloutHistoryList + plural: rollouthistories + singular: rollouthistory + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Phase indicates phase of RolloutHistory, such as pending, progressing, + completed + jsonPath: .status.phase + name: PHASE + type: string + - description: CanaryStepIndex indicates the current step + jsonPath: .status.canaryStepIndex + name: CURRENT_STEP + type: integer + - description: CanaryStepState indicates state of this rollout revision, such + as init, pending, update, terminated, completed, cancelled, whick is upon + rollout canary_step_state + jsonPath: .status.canaryStepState + name: CURRENT_STATE + type: string + - description: Message provides details on why the rollout is in its current phase + jsonPath: .status.rolloutState.message + name: MESSAGE + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: RolloutHistory is the Schema for the rollouthistories API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RolloutHistorySpec defines the desired state of RolloutHistory + properties: + rolloutWrapper: + description: RolloutWrapper indicates information of the rollout related + with rollouthistory + properties: + name: + description: Name indicates the rollout name + type: string + rollout: + description: Rollout indecates the related rollout + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RolloutSpec defines the desired state of Rollout + properties: + objectRef: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired + state of cluster Important: Run "make" to regenerate + code after modifying this file ObjectRef indicates workload' + properties: + workloadRef: + description: WorkloadRef contains enough information + to let you identify a workload for Rollout Batch + release of the bypass + properties: + apiVersion: + description: API Version of the referent + type: string + kind: + description: Kind of the referent + type: string + name: + description: Name of the referent + type: string + required: + - apiVersion + - kind + - name + type: object + type: object + rolloutID: + description: RolloutID should be changed before each workload + revision publication. It is to distinguish consecutive + multiple workload publications and rollout progress. + type: string + strategy: + description: rollout strategy + properties: + canary: + description: CanaryStrategy defines parameters for + a Replica Based Canary + properties: + steps: + description: Steps define the order of phases + to execute release in batches(20%, 40%, 60%, + 80%, 100%) + items: + description: CanaryStep defines a step of a + canary workload. + properties: + pause: + description: Pause defines a pause stage + for a rollout, manual or auto + properties: + duration: + description: Duration the amount of + time to wait before moving to the + next step. + format: int32 + type: integer + type: object + replicas: + anyOf: + - type: integer + - type: string + description: 'Replicas is the number of + expected canary pods in this batch it + can be an absolute number (ex: 5) or a + percentage of total pods.' + x-kubernetes-int-or-string: true + weight: + format: int32 + type: integer + type: object + type: array + trafficRoutings: + description: TrafficRoutings hosts all the supported + service meshes supported to enable more fine-grained + traffic routing todo current only support one + TrafficRouting + items: + description: TrafficRouting hosts all the different + configuration for supported service meshes + to enable more fine-grained traffic routing + properties: + gateway: + description: Gateway holds Gateway specific + configuration to route traffic Gateway + configuration only supports >= v0.4.0 + (v1alpha2). + properties: + httpRouteName: + description: HTTPRouteName refers to + the name of an `HTTPRoute` resource + in the same namespace as the `Rollout` + type: string + type: object + gracePeriodSeconds: + description: Optional duration in seconds + the traffic provider(e.g. nginx ingress + controller) consumes the service, ingress + configuration changes gracefully. + format: int32 + type: integer + ingress: + description: Ingress holds Ingress specific + configuration to route traffic, e.g. Nginx, + Alb. + properties: + classType: + description: ClassType refers to the + class type of an `Ingress`, e.g. Nginx. + Default is Nginx + type: string + name: + description: Name refers to the name + of an `Ingress` resource in the same + namespace as the `Rollout` + type: string + required: + - name + type: object + service: + description: Service holds the name of a + service which selects pods with stable + version and don't select any pods with + canary version. + type: string + required: + - service + type: object + type: array + type: object + paused: + description: Paused indicates that the Rollout is + paused. Default value is false + type: boolean + type: object + required: + - objectRef + - strategy + type: object + status: + description: RolloutStatus defines the observed state of Rollout + properties: + canaryStatus: + description: Canary describes the state of the canary + rollout + properties: + canaryReadyReplicas: + description: CanaryReadyReplicas the numbers of ready + canary revision pods + format: int32 + type: integer + canaryReplicas: + description: CanaryReplicas the numbers of canary + revision pods + format: int32 + type: integer + canaryRevision: + description: CanaryRevision is calculated by rollout + based on podTemplateHash, and the internal logic + flow uses It may be different from rs podTemplateHash + in different k8s versions, so it cannot be used + as service selector label + type: string + canaryService: + description: CanaryService holds the name of a service + which selects pods with canary version and don't + select any pods with stable version. + type: string + currentStepIndex: + description: CurrentStepIndex defines the current + step of the rollout is on. If the current step index + is null, the controller will execute the rollout. + format: int32 + type: integer + currentStepState: + type: string + lastUpdateTime: + format: date-time + type: string + message: + type: string + observedRolloutID: + description: ObservedRolloutID will record the newest + spec.RolloutID if status.canaryRevision equals to + workload.updateRevision + type: string + observedWorkloadGeneration: + description: observedWorkloadGeneration is the most + recent generation observed for this Rollout ref + workload generation. + format: int64 + type: integer + podTemplateHash: + description: pod template hash is used as service + selector label + type: string + rolloutHash: + description: RolloutHash from rollout.spec object + type: string + required: + - canaryReadyReplicas + - canaryReplicas + - canaryService + - currentStepState + - podTemplateHash + type: object + conditions: + description: Conditions a list of conditions a rollout + can have. + items: + description: RolloutCondition describes the state of + a rollout at a certain point. + properties: + lastTransitionTime: + description: Last time the condition transitioned + from one status to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating + details about the transition. + type: string + reason: + description: The reason for the condition's last + transition. + type: string + status: + description: Phase of the condition, one of True, + False, Unknown. + type: string + type: + description: Type of rollout condition. + type: string + required: + - message + - reason + - status + - type + type: object + type: array + message: + description: Message provides details on why the rollout + is in its current phase + type: string + observedGeneration: + description: observedGeneration is the most recent generation + observed for this Rollout. + format: int64 + type: integer + phase: + description: BlueGreenStatus *BlueGreenStatus `json:"blueGreenStatus,omitempty"` + Phase is the rollout phase. + type: string + stableRevision: + description: CanaryRevision the hash of the canary pod + template CanaryRevision string `json:"canaryRevision,omitempty"` + StableRevision indicates the revision pods that has + successfully rolled out + type: string + type: object + type: object + type: object + serviceWrapper: + description: ServiceWrapper indicates information of the service related + with workload + properties: + name: + description: Name indicates the service name + type: string + service: + description: Service indicates the service + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: Spec defines the behavior of a service. https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + allocateLoadBalancerNodePorts: + description: allocateLoadBalancerNodePorts defines if + NodePorts will be automatically allocated for services + with type LoadBalancer. Default is "true". It may be + set to "false" if the cluster load-balancer does not + rely on NodePorts. If the caller requests specific + NodePorts (by specifying a value), those requests will + be respected, regardless of this field. This field may + only be set for services with type LoadBalancer and + will be cleared if the type is changed to any other + type. This field is beta-level and is only honored by + servers that enable the ServiceLBNodePortControl feature. + type: boolean + clusterIP: + description: 'clusterIP is the IP address of the service + and is usually assigned randomly. If an address is specified + manually, is in-range (as per system configuration), + and is not in use, it will be allocated to the service; + otherwise creation of the service will fail. This field + may not be changed through updates unless the type field + is also being changed to ExternalName (which requires + this field to be blank) or the type field is being changed + from ExternalName (in which case this field may optionally + be specified, as describe above). Valid values are + "None", empty string (""), or a valid IP address. Setting + this to "None" makes a "headless service" (no virtual + IP), which is useful when direct endpoint connections + are preferred and proxying is not required. Only applies + to types ClusterIP, NodePort, and LoadBalancer. If this + field is specified when creating a Service of type ExternalName, + creation will fail. This field will be wiped when updating + a Service to type ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + clusterIPs: + description: "ClusterIPs is a list of IP addresses assigned + to this service, and are usually assigned randomly. + \ If an address is specified manually, is in-range (as + per system configuration), and is not in use, it will + be allocated to the service; otherwise creation of the + service will fail. This field may not be changed through + updates unless the type field is also being changed + to ExternalName (which requires this field to be empty) + or the type field is being changed from ExternalName + (in which case this field may optionally be specified, + as describe above). Valid values are \"None\", empty + string (\"\"), or a valid IP address. Setting this + to \"None\" makes a \"headless service\" (no virtual + IP), which is useful when direct endpoint connections + are preferred and proxying is not required. Only applies + to types ClusterIP, NodePort, and LoadBalancer. If this + field is specified when creating a Service of type ExternalName, + creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not + specified, it will be initialized from the clusterIP + field. If this field is specified, clients must ensure + that clusterIPs[0] and clusterIP have the same value. + \n Unless the \"IPv6DualStack\" feature gate is enabled, + this field is limited to one value, which must be the + same as the clusterIP field. If the feature gate is + enabled, this field may hold a maximum of two entries + (dual-stack IPs, in either order). These IPs must correspond + to the values of the ipFamilies field. Both clusterIPs + and ipFamilies are governed by the ipFamilyPolicy field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: externalIPs is a list of IP addresses for + which nodes in the cluster will also accept traffic + for this service. These IPs are not managed by Kubernetes. The + user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external + load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that + discovery mechanisms will return as an alias for this + service (e.g. a DNS CNAME record). No proxying will + be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` + to be "ExternalName". + type: string + externalTrafficPolicy: + description: externalTrafficPolicy denotes if this Service + desires to route external traffic to node-local or cluster-wide + endpoints. "Local" preserves the client source IP and + avoids a second hop for LoadBalancer and Nodeport type + services, but risks potentially imbalanced traffic spreading. + "Cluster" obscures the client source IP and may cause + a second hop to another node, but should have good overall + load-spreading. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. This only applies when type + is set to LoadBalancer and externalTrafficPolicy is + set to Local. If a value is specified, is in-range, + and is not in use, it will be used. If not specified, + a value will be automatically allocated. External systems + (e.g. load-balancers) can use this port to determine + if a given node holds endpoints for this service or + not. If this field is specified when creating a Service + which does not need it, creation will fail. This field + will be wiped when updating a Service to no longer need + it (e.g. changing type). + format: int32 + type: integer + internalTrafficPolicy: + description: InternalTrafficPolicy specifies if the cluster + internal traffic should be routed to all endpoints or + node-local endpoints only. "Cluster" routes internal + traffic to a Service to all endpoints. "Local" routes + traffic to node-local endpoints only, traffic is dropped + if no node-local endpoints are ready. The default value + is "Cluster". + type: string + ipFamilies: + description: "IPFamilies is a list of IP families (e.g. + IPv4, IPv6) assigned to this service, and is gated by + the \"IPv6DualStack\" feature gate. This field is usually + assigned automatically based on cluster configuration + and the ipFamilyPolicy field. If this field is specified + manually, the requested family is available in the cluster, + and ipFamilyPolicy allows it, it will be used; otherwise + creation of the service will fail. This field is conditionally + mutable: it allows for adding or removing a secondary + IP family, but it does not allow changing the primary + IP family of the Service. Valid values are \"IPv4\" + and \"IPv6\". This field only applies to Services of + types ClusterIP, NodePort, and LoadBalancer, and does + apply to \"headless\" services. This field will be + wiped when updating a Service to type ExternalName. + \n This field may hold a maximum of two entries (dual-stack + families, in either order). These families must correspond + to the values of the clusterIPs field, if specified. + Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy + field." + items: + description: IPFamily represents the IP Family (IPv4 + or IPv6). This type is used to express the family + of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness + requested or required by this Service, and is gated + by the "IPv6DualStack" feature gate. If there is no + value provided, then this field will be set to SingleStack. + Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured + clusters or a single IP family on single-stack clusters), + or "RequireDualStack" (two IP families on dual-stack + configured clusters, otherwise fail). The ipFamilies + and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type + ExternalName. + type: string + loadBalancerClass: + description: loadBalancerClass is the class of the load + balancer implementation this Service belongs to. If + specified, the value of this field must be a label-style + identifier, with an optional prefix, e.g. "internal-vip" + or "example.com/internal-vip". Unprefixed names are + reserved for end-users. This field can only be set when + the Service type is 'LoadBalancer'. If not set, the + default load balancer implementation is used, today + this is typically done through the cloud provider integration, + but should apply for any default implementation. If + set, it is assumed that a load balancer implementation + is watching for Services with a matching class. Any + default load balancer implementation (e.g. cloud providers) + should ignore Services that set this field. This field + can only be set when creating or updating a Service + to type 'LoadBalancer'. Once set, it can not be changed. + This field will be wiped when a service is updated to + a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer + LoadBalancer will get created with the IP specified + in this field. This feature depends on whether the underlying + cloud-provider supports specifying the loadBalancerIP + when a load balancer is created. This field will be + ignored if the cloud-provider does not support the feature.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client + IPs. This field will be ignored if the cloud-provider + does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this + service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + appProtocol: + description: The application protocol for this port. + This field follows standard Kubernetes label syntax. + Un-prefixed names are reserved for IANA standard + service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). + Non-standard protocols should use prefixed names + such as mycompany.com/my-custom-protocol. + type: string + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field + in the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this + service is exposed when type is NodePort or LoadBalancer. Usually + assigned by the system. If a value is specified, + in-range, and not in use it will be used, otherwise + the operation will fail. If not specified, a + port will be allocated if this Service requires + one. If this field is specified when creating + a Service which does not need it, creation will + fail. This field will be wiped when updating a + Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this + service. + format: int32 + type: integer + protocol: + default: TCP + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access + on the pods targeted by the service. Number must + be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a + named port in the target Pod''s container ports. + If this is not specified, the value of the ''port'' + field is used (an identity map). This field is + ignored for services with clusterIP=None, and + should be omitted or set equal to the ''port'' + field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: publishNotReadyAddresses indicates that any + agent which deals with endpoints for this Service should + disregard any indications of ready/not-ready. The primary + use case for setting this field is for a StatefulSet's + Headless Service to propagate SRV DNS records for its + Pods for the purpose of peer discovery. The Kubernetes + controllers that generate Endpoints and EndpointSlice + resources for Services interpret this to mean that all + endpoints are considered "ready" even if the Pods themselves + are not. Agents which consume only Kubernetes generated + endpoints through the Endpoints or EndpointSlice resources + can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label + keys and values matching this selector. If empty or + not present, the service is assumed to have an external + process managing its endpoints, which Kubernetes will + not modify. Only applies to types ClusterIP, NodePort, + and LoadBalancer. Ignored if type is ExternalName. More + info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to + maintain session affinity. Enable client IP based session + affinity. Must be ClientIP or None. Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 + hours). + format: int32 + type: integer + type: object + type: object + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, + ClusterIP, NodePort, and LoadBalancer. "ClusterIP" allocates + a cluster-internal IP address for load-balancing to + endpoints. Endpoints are determined by the selector + or if that is not specified, by manual construction + of an Endpoints object or EndpointSlice objects. If + clusterIP is "None", no virtual IP is allocated and + the endpoints are published as a set of endpoints rather + than a virtual IP. "NodePort" builds on ClusterIP and + allocates a port on every node which routes to the same + endpoints as the clusterIP. "LoadBalancer" builds on + NodePort and creates an external load-balancer (if supported + in the current cloud) which routes to the same endpoints + as the clusterIP. "ExternalName" aliases this service + to the specified externalName. Several other fields + do not apply to ExternalName services. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + status: + description: 'Most recently observed status of the service. + Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + conditions: + description: Current service state + items: + description: "Condition contains details for one aspect + of the current state of this API Resource. --- This + struct is intended for direct use as an array at the + field path .status.conditions. For example, type + FooStatus struct{ // Represents the observations + of a foo's current state. // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\" + \ // +patchMergeKey=type // +patchStrategy=merge + \ // +listType=map // +listMapKey=type Conditions + []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` + \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time + the condition transitioned from one status to + another. This should be when the underlying condition + changed. If that is not known, then using the + time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message + indicating details about the transition. This + may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, + if .metadata.generation is currently 12, but the + .status.conditions[x].observedGeneration is 9, + the condition is out of date with respect to the + current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier + indicating the reason for the condition's last + transition. Producers of specific condition types + may define expected values and meanings for this + field, and whether the values are considered a + guaranteed API. The value should be a CamelCase + string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, + False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in + foo.example.com/CamelCase. --- Many .condition.type + values are consistent across resources like Available, + but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to + deconflict is important. The regex it matches + is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: LoadBalancer contains the current status + of the load-balancer, if one is present. + properties: + ingress: + description: Ingress is a list containing ingress + points for the load-balancer. Traffic intended for + the service should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the + status of a load-balancer ingress point: traffic + intended for the service should be sent to an + ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer + ingress points that are DNS based (typically + AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress + points that are IP based (typically GCE or + OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service + ports If used, every port defined in the service + should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem + with the service port The format of + the error shall comply with the following + rules: - built-in error values shall + be specified in this file and those + shall use CamelCase names - cloud + provider specific error values must + have names that comply with the format + foo.example.com/CamelCase. --- The regex + it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of + the service port of which status is + recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol + of the service port of which status + is recorded here The supported values + are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + type: object + type: object + type: object + trafficRoutingWrapper: + description: TrafficRoutingWrapper indicates information of traffic + route related with workload + properties: + httpRouteWrapper: + description: HTTPRouteWrapper indacates information of Gateway + API + properties: + httpRoute: + description: HTTPRoute indicates the HTTPRoute + properties: + apiVersion: + description: 'APIVersion defines the versioned schema + of this representation of an object. Servers should + convert recognized schemas to the latest internal value, + and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the + REST resource this object represents. Servers may infer + this from the endpoint the client submits requests to. + Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of HTTPRoute. + properties: + hostnames: + description: "Hostnames defines a set of hostname + that should match against the HTTP Host header to + select a HTTPRoute to process the request. This + matches the RFC 1123 definition of a hostname with + 2 notable exceptions: \n 1. IPs are not allowed. + 2. A hostname may be prefixed with a wildcard label + (`*.`). The wildcard label must appear by itself + as the first label. \n If a hostname is specified + by both the Listener and HTTPRoute, there must be + at least one intersecting hostname for the HTTPRoute + to be attached to the Listener. For example: \n + * A Listener with `test.example.com` as the hostname + matches HTTPRoutes that have either not specified + any hostnames, or have specified at least one + of `test.example.com` or `*.example.com`. * A Listener + with `*.example.com` as the hostname matches HTTPRoutes + \ that have either not specified any hostnames + or have specified at least one hostname that matches + the Listener hostname. For example, `test.example.com` + and `*.example.com` would both match. On the other + \ hand, `example.com` and `test.example.net` would + not match. \n If both the Listener and HTTPRoute + have specified hostnames, any HTTPRoute hostnames + that do not match the Listener hostname MUST be + ignored. For example, if a Listener specified `*.example.com`, + and the HTTPRoute specified `test.example.com` and + `test.example.net`, `test.example.net` must not + be considered for a match. \n If both the Listener + and HTTPRoute have specified hostnames, and none + match with the criteria above, then the HTTPRoute + is not accepted. The implementation must raise an + 'Accepted' Condition with a status of `False` in + the corresponding RouteParentStatus. \n Support: + Core" + items: + description: "Hostname is the fully qualified domain + name of a network host. This matches the RFC 1123 + definition of a hostname with 2 notable exceptions: + \n 1. IPs are not allowed. 2. A hostname may be + prefixed with a wildcard label (`*.`). The wildcard + \ label must appear by itself as the first label. + \n Hostname can be \"precise\" which is a domain + name without the terminating dot of a network + host (e.g. \"foo.example.com\") or \"wildcard\", + which is a domain name prefixed with a single + wildcard label (e.g. `*.example.com`). \n Note + that as per RFC1035 and RFC1123, a *label* must + consist of lower case alphanumeric characters + or '-', and must start and end with an alphanumeric + character. No other punctuation is allowed." + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + maxItems: 16 + type: array + parentRefs: + description: "ParentRefs references the resources + (usually Gateways) that a Route wants to be attached + to. Note that the referenced parent resource needs + to allow this for the attachment to be complete. + For Gateways, that means the Gateway needs to allow + attachment from Routes of this kind and namespace. + \n The only kind of parent resource with \"Core\" + support is Gateway. This API may be extended in + the future to support additional kinds of parent + resources such as one of the route kinds. \n It + is invalid to reference an identical parent more + than once. It is valid to reference multiple distinct + sections within the same parent resource, such as + 2 Listeners within a Gateway. \n It is possible + to separately reference multiple distinct objects + that may be collapsed by an implementation. For + example, some implementations may choose to merge + compatible Gateway Listeners together. If that is + the case, the list of routes attached to those resources + should also be merged." + items: + description: "ParentRef identifies an API object + (usually a Gateway) that can be considered a parent + of this resource (usually a route). The only kind + of parent resource with \"Core\" support is Gateway. + This API may be extended in the future to support + additional kinds of parent resources, such as + HTTPRoute. \n The API object must be valid in + the cluster; the Group and Kind must be registered + in the cluster for this reference to be valid. + \n References to objects with invalid Group and + Kind are not valid, and must be rejected by the + implementation, with appropriate Conditions set + on the containing object." + properties: + group: + default: gateway.networking.k8s.io + description: "Group is the group of the referent. + \n Support: Core" + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: "Kind is kind of the referent. + \n Support: Core (Gateway) Support: Custom + (Other Resources)" + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: "Name is the name of the referent. + \n Support: Core" + maxLength: 253 + minLength: 1 + type: string + namespace: + description: "Namespace is the namespace of + the referent. When unspecified (or empty string), + this refers to the local namespace of the + Route. \n Support: Core" + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + sectionName: + description: "SectionName is the name of a section + within the target resource. In the following + resources, SectionName is interpreted as the + following: \n * Gateway: Listener Name \n + Implementations MAY choose to support attaching + Routes to other resources. If that is the + case, they MUST clearly document how SectionName + is interpreted. \n When unspecified (empty + string), this will reference the entire resource. + For the purpose of status, an attachment is + considered successful if at least one section + in the parent resource accepts it. For example, + Gateway listeners can restrict which Routes + can attach to them by Route kind, namespace, + or hostname. If 1 of 2 Gateway listeners accept + attachment from the referencing Route, the + Route MUST be considered successfully attached. + If no Gateway listeners accept attachment + from this Route, the Route MUST be considered + detached from the Gateway. \n Support: Core" + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + maxItems: 32 + type: array + rules: + default: + - matches: + - path: + type: PathPrefix + value: / + description: Rules are a list of HTTP matchers, filters + and actions. + items: + description: HTTPRouteRule defines semantics for + matching an HTTP request based on conditions (matches), + processing it (filters), and forwarding the request + to an API object (backendRefs). + properties: + backendRefs: + description: "If unspecified or invalid (refers + to a non-existent resource or a Service with + no endpoints), the rule performs no forwarding. + If there are also no filters specified that + would result in a response being sent, a HTTP + 503 status code is returned. 503 responses + must be sent so that the overall weight is + respected; if an invalid backend is requested + to have 80% of requests, then 80% of requests + must get a 503 instead. \n Support: Core for + Kubernetes Service Support: Custom for any + other resource \n Support for weight: Core" + items: + description: HTTPBackendRef defines how a + HTTPRoute should forward an HTTP request. + properties: + filters: + description: "Filters defined at this + level should be executed if and only + if the request is being forwarded to + the backend defined here. \n Support: + Custom (For broader support of filters, + use the Filters field in HTTPRouteRule.)" + items: + description: HTTPRouteFilter defines + processing steps that must be completed + during the request or response lifecycle. + HTTPRouteFilters are meant as an extension + point to express processing that may + be done in Gateway implementations. + Some examples include request or response + modification, implementing authentication + strategies, rate-limiting, and traffic + shaping. API guarantee/conformance + is defined based on the type of the + filter. + properties: + extensionRef: + description: "ExtensionRef is an + optional, implementation-specific + extension to the \"filter\" behavior. + \ For example, resource \"myroutefilter\" + in group \"networking.example.net\"). + ExtensionRef MUST NOT be used + for core and extended filters. + \n Support: Implementation-specific" + properties: + group: + description: Group is the group + of the referent. For example, + "networking.k8s.io". When + unspecified (empty string), + core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of + the referent. For example + "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name + of the referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + requestHeaderModifier: + description: "RequestHeaderModifier + defines a schema for a filter + that modifies request headers. + \n Support: Core" + properties: + add: + description: "Add adds the given + header(s) (name, value) to + the request before the action. + It appends to any existing + values associated with the + header name. \n Input: GET + /foo HTTP/1.1 my-header: + foo \n Config: add: - + name: \"my-header\" value: + \"bar\" \n Output: GET /foo + HTTP/1.1 my-header: foo + \ my-header: bar" + items: + description: HTTPHeader represents + an HTTP Header name and + value as defined by RFC + 7230. + properties: + name: + description: "Name is + the name of the HTTP + Header to be matched. + Name matching MUST be + case insensitive. (See + https://tools.ietf.org/html/rfc7230#section-3.2). + \n If multiple entries + specify equivalent header + names, the first entry + with an equivalent name + MUST be considered for + a match. Subsequent + entries with an equivalent + header name MUST be + ignored. Due to the + case-insensitivity of + header names, \"foo\" + and \"Foo\" are considered + equivalent." + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is + the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: "Remove the given + header(s) from the HTTP request + before the action. The value + of Remove is a list of HTTP + header names. Note that the + header names are case-insensitive + (see https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + \n Input: GET /foo HTTP/1.1 + \ my-header1: foo my-header2: + bar my-header3: baz \n Config: + \ remove: [\"my-header1\", + \"my-header3\"] \n Output: + \ GET /foo HTTP/1.1 my-header2: + bar" + items: + type: string + maxItems: 16 + type: array + set: + description: "Set overwrites + the request with the given + header (name, value) before + the action. \n Input: GET + /foo HTTP/1.1 my-header: + foo \n Config: set: - + name: \"my-header\" value: + \"bar\" \n Output: GET /foo + HTTP/1.1 my-header: bar" + items: + description: HTTPHeader represents + an HTTP Header name and + value as defined by RFC + 7230. + properties: + name: + description: "Name is + the name of the HTTP + Header to be matched. + Name matching MUST be + case insensitive. (See + https://tools.ietf.org/html/rfc7230#section-3.2). + \n If multiple entries + specify equivalent header + names, the first entry + with an equivalent name + MUST be considered for + a match. Subsequent + entries with an equivalent + header name MUST be + ignored. Due to the + case-insensitivity of + header names, \"foo\" + and \"Foo\" are considered + equivalent." + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is + the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + requestMirror: + description: "RequestMirror defines + a schema for a filter that mirrors + requests. Requests are sent to + the specified destination, but + responses from that destination + are ignored. \n Support: Extended" + properties: + backendRef: + description: "BackendRef references + a resource where mirrored + requests are sent. \n If the + referent cannot be found, + this BackendRef is invalid + and must be dropped from the + Gateway. The controller must + ensure the \"ResolvedRefs\" + condition on the Route status + is set to `status: False` + and not configure this backend + in the underlying implementation. + \n If there is a cross-namespace + reference to an *existing* + object that is not allowed + by a ReferencePolicy, the + controller must ensure the + \"ResolvedRefs\" condition + on the Route is set to `status: + False`, with the \"RefNotPermitted\" + reason and not configure this + backend in the underlying + implementation. \n In either + error case, the Message of + the `ResolvedRefs` Condition + should be used to provide + more detail about the problem. + \n Support: Extended for Kubernetes + Service Support: Custom for + any other resource" + properties: + group: + default: "" + description: Group is the + group of the referent. + For example, "networking.k8s.io". + When unspecified (empty + string), core API group + is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kind is kind + of the referent. For example + "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the + name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: "Namespace + is the namespace of the + backend. When unspecified, + the local namespace is + inferred. \n Note that + when a namespace is specified, + a ReferencePolicy object + is required in the referent + namespace to allow that + namespace's owner to accept + the reference. See the + ReferencePolicy documentation + for details. \n Support: + Core" + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Port specifies + the destination port number + to use for this resource. + Port is required when + the referent is a Kubernetes + Service. For other resources, + destination port might + be derived from the referent + resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + required: + - backendRef + type: object + requestRedirect: + description: "RequestRedirect defines + a schema for a filter that responds + to the request with an HTTP redirection. + \n Support: Core" + properties: + hostname: + description: "Hostname is the + hostname to be used in the + value of the `Location` header + in the response. When empty, + the hostname of the request + is used. \n Support: Core" + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + port: + description: "Port is the port + to be used in the value of + the `Location` header in the + response. When empty, port + (if specified) of the request + is used. \n Support: Extended" + format: int32 + maximum: 65535 + minimum: 1 + type: integer + scheme: + description: "Scheme is the + scheme to be used in the value + of the `Location` header in + the response. When empty, + the scheme of the request + is used. \n Support: Extended" + enum: + - http + - https + type: string + statusCode: + default: 302 + description: "StatusCode is + the HTTP status code to be + used in response. \n Support: + Core" + enum: + - 301 + - 302 + type: integer + type: object + type: + description: "Type identifies the + type of filter to apply. As with + other API fields, types are classified + into three conformance levels: + \n - Core: Filter types and their + corresponding configuration defined + by \"Support: Core\" in this + package, e.g. \"RequestHeaderModifier\". + All implementations must support + core filters. \n - Extended: Filter + types and their corresponding + configuration defined by \"Support: + Extended\" in this package, e.g. + \"RequestMirror\". Implementers + \ are encouraged to support extended + filters. \n - Custom: Filters + that are defined and supported + by specific vendors. In the + future, filters showing convergence + in behavior across multiple implementations + will be considered for inclusion + in extended or core conformance + levels. Filter-specific configuration + for such filters is specified + using the ExtensionRef field. + `Type` should be set to \"ExtensionRef\" + for custom filters. \n Implementers + are encouraged to define custom + implementation types to extend + the core API with implementation-specific + behavior. \n If a reference to + a custom filter type cannot be + resolved, the filter MUST NOT + be skipped. Instead, requests + that would have been processed + by that filter MUST receive a + HTTP error response." + enum: + - RequestHeaderModifier + - RequestMirror + - RequestRedirect + - ExtensionRef + type: string + required: + - type + type: object + maxItems: 16 + type: array + group: + default: "" + description: Group is the group of the + referent. For example, "networking.k8s.io". + When unspecified (empty string), core + API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kind is kind of the referent. + For example "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: "Namespace is the namespace + of the backend. When unspecified, the + local namespace is inferred. \n Note + that when a namespace is specified, + a ReferencePolicy object is required + in the referent namespace to allow that + namespace's owner to accept the reference. + See the ReferencePolicy documentation + for details. \n Support: Core" + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Port specifies the destination + port number to use for this resource. + Port is required when the referent is + a Kubernetes Service. For other resources, + destination port might be derived from + the referent resource or this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + weight: + default: 1 + description: "Weight specifies the proportion + of requests forwarded to the referenced + backend. This is computed as weight/(sum + of all weights in this BackendRefs list). + For non-zero values, there may be some + epsilon from the exact proportion defined + here depending on the precision an implementation + supports. Weight is not a percentage + and the sum of weights does not need + to equal 100. \n If only one backend + is specified and it has a weight greater + than 0, 100% of the traffic is forwarded + to that backend. If weight is set to + 0, no traffic should be forwarded for + this entry. If unspecified, weight defaults + to 1. \n Support for this field varies + based on the context where used." + format: int32 + maximum: 1000000 + minimum: 0 + type: integer + required: + - name + type: object + maxItems: 16 + type: array + filters: + description: "Filters define the filters that + are applied to requests that match this rule. + \n The effects of ordering of multiple behaviors + are currently unspecified. This can change + in the future based on feedback during the + alpha stage. \n Conformance-levels at this + level are defined based on the type of filter: + \n - ALL core filters MUST be supported by + all implementations. - Implementers are encouraged + to support extended filters. - Implementation-specific + custom filters have no API guarantees across + \ implementations. \n Specifying a core filter + multiple times has unspecified or custom conformance. + \n Support: Core" + items: + description: HTTPRouteFilter defines processing + steps that must be completed during the + request or response lifecycle. HTTPRouteFilters + are meant as an extension point to express + processing that may be done in Gateway implementations. + Some examples include request or response + modification, implementing authentication + strategies, rate-limiting, and traffic shaping. + API guarantee/conformance is defined based + on the type of the filter. + properties: + extensionRef: + description: "ExtensionRef is an optional, + implementation-specific extension to + the \"filter\" behavior. For example, + resource \"myroutefilter\" in group + \"networking.example.net\"). ExtensionRef + MUST NOT be used for core and extended + filters. \n Support: Implementation-specific" + properties: + group: + description: Group is the group of + the referent. For example, "networking.k8s.io". + When unspecified (empty string), + core API group is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + description: Kind is kind of the referent. + For example "HTTPRoute" or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name of the + referent. + maxLength: 253 + minLength: 1 + type: string + required: + - group + - kind + - name + type: object + requestHeaderModifier: + description: "RequestHeaderModifier defines + a schema for a filter that modifies + request headers. \n Support: Core" + properties: + add: + description: "Add adds the given header(s) + (name, value) to the request before + the action. It appends to any existing + values associated with the header + name. \n Input: GET /foo HTTP/1.1 + \ my-header: foo \n Config: add: + \ - name: \"my-header\" value: + \"bar\" \n Output: GET /foo HTTP/1.1 + \ my-header: foo my-header: bar" + items: + description: HTTPHeader represents + an HTTP Header name and value + as defined by RFC 7230. + properties: + name: + description: "Name is the name + of the HTTP Header to be matched. + Name matching MUST be case + insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + \n If multiple entries specify + equivalent header names, the + first entry with an equivalent + name MUST be considered for + a match. Subsequent entries + with an equivalent header + name MUST be ignored. Due + to the case-insensitivity + of header names, \"foo\" and + \"Foo\" are considered equivalent." + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value + of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + remove: + description: "Remove the given header(s) + from the HTTP request before the + action. The value of Remove is a + list of HTTP header names. Note + that the header names are case-insensitive + (see https://datatracker.ietf.org/doc/html/rfc2616#section-4.2). + \n Input: GET /foo HTTP/1.1 my-header1: + foo my-header2: bar my-header3: + baz \n Config: remove: [\"my-header1\", + \"my-header3\"] \n Output: GET + /foo HTTP/1.1 my-header2: bar" + items: + type: string + maxItems: 16 + type: array + set: + description: "Set overwrites the request + with the given header (name, value) + before the action. \n Input: GET + /foo HTTP/1.1 my-header: foo \n + Config: set: - name: \"my-header\" + \ value: \"bar\" \n Output: GET + /foo HTTP/1.1 my-header: bar" + items: + description: HTTPHeader represents + an HTTP Header name and value + as defined by RFC 7230. + properties: + name: + description: "Name is the name + of the HTTP Header to be matched. + Name matching MUST be case + insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + \n If multiple entries specify + equivalent header names, the + first entry with an equivalent + name MUST be considered for + a match. Subsequent entries + with an equivalent header + name MUST be ignored. Due + to the case-insensitivity + of header names, \"foo\" and + \"Foo\" are considered equivalent." + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value + of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + requestMirror: + description: "RequestMirror defines a + schema for a filter that mirrors requests. + Requests are sent to the specified destination, + but responses from that destination + are ignored. \n Support: Extended" + properties: + backendRef: + description: "BackendRef references + a resource where mirrored requests + are sent. \n If the referent cannot + be found, this BackendRef is invalid + and must be dropped from the Gateway. + The controller must ensure the \"ResolvedRefs\" + condition on the Route status is + set to `status: False` and not configure + this backend in the underlying implementation. + \n If there is a cross-namespace + reference to an *existing* object + that is not allowed by a ReferencePolicy, + the controller must ensure the \"ResolvedRefs\" + \ condition on the Route is set + to `status: False`, with the \"RefNotPermitted\" + reason and not configure this backend + in the underlying implementation. + \n In either error case, the Message + of the `ResolvedRefs` Condition + should be used to provide more detail + about the problem. \n Support: Extended + for Kubernetes Service Support: + Custom for any other resource" + properties: + group: + default: "" + description: Group is the group + of the referent. For example, + "networking.k8s.io". When unspecified + (empty string), core API group + is inferred. + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Service + description: Kind is kind of the + referent. For example "HTTPRoute" + or "Service". + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: Name is the name + of the referent. + maxLength: 253 + minLength: 1 + type: string + namespace: + description: "Namespace is the + namespace of the backend. When + unspecified, the local namespace + is inferred. \n Note that when + a namespace is specified, a + ReferencePolicy object is required + in the referent namespace to + allow that namespace's owner + to accept the reference. See + the ReferencePolicy documentation + for details. \n Support: Core" + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: Port specifies the + destination port number to use + for this resource. Port is required + when the referent is a Kubernetes + Service. For other resources, + destination port might be derived + from the referent resource or + this field. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + required: + - name + type: object + required: + - backendRef + type: object + requestRedirect: + description: "RequestRedirect defines + a schema for a filter that responds + to the request with an HTTP redirection. + \n Support: Core" + properties: + hostname: + description: "Hostname is the hostname + to be used in the value of the `Location` + header in the response. When empty, + the hostname of the request is used. + \n Support: Core" + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + port: + description: "Port is the port to + be used in the value of the `Location` + header in the response. When empty, + port (if specified) of the request + is used. \n Support: Extended" + format: int32 + maximum: 65535 + minimum: 1 + type: integer + scheme: + description: "Scheme is the scheme + to be used in the value of the `Location` + header in the response. When empty, + the scheme of the request is used. + \n Support: Extended" + enum: + - http + - https + type: string + statusCode: + default: 302 + description: "StatusCode is the HTTP + status code to be used in response. + \n Support: Core" + enum: + - 301 + - 302 + type: integer + type: object + type: + description: "Type identifies the type + of filter to apply. As with other API + fields, types are classified into three + conformance levels: \n - Core: Filter + types and their corresponding configuration + defined by \"Support: Core\" in this + package, e.g. \"RequestHeaderModifier\". + All implementations must support core + filters. \n - Extended: Filter types + and their corresponding configuration + defined by \"Support: Extended\" in + this package, e.g. \"RequestMirror\". + Implementers are encouraged to support + extended filters. \n - Custom: Filters + that are defined and supported by specific + vendors. In the future, filters showing + convergence in behavior across multiple + \ implementations will be considered + for inclusion in extended or core conformance + levels. Filter-specific configuration + for such filters is specified using + the ExtensionRef field. `Type` should + be set to \"ExtensionRef\" for custom + filters. \n Implementers are encouraged + to define custom implementation types + to extend the core API with implementation-specific + behavior. \n If a reference to a custom + filter type cannot be resolved, the + filter MUST NOT be skipped. Instead, + requests that would have been processed + by that filter MUST receive a HTTP error + response." + enum: + - RequestHeaderModifier + - RequestMirror + - RequestRedirect + - ExtensionRef + type: string + required: + - type + type: object + maxItems: 16 + type: array + matches: + default: + - path: + type: PathPrefix + value: / + description: "Matches define conditions used + for matching the rule against incoming HTTP + requests. Each match is independent, i.e. + this rule will be matched if **any** one of + the matches is satisfied. \n For example, + take the following matches configuration: + \n ``` matches: - path: value: \"/foo\" + \ headers: - name: \"version\" value: + \"v2\" - path: value: \"/v2/foo\" ``` + \n For a request to match against this rule, + a request must satisfy EITHER of the two conditions: + \n - path prefixed with `/foo` AND contains + the header `version: v2` - path prefix of + `/v2/foo` \n See the documentation for HTTPRouteMatch + on how to specify multiple match conditions + that should be ANDed together. \n If no matches + are specified, the default is a prefix path + match on \"/\", which has the effect of matching + every HTTP request. \n Proxy or Load Balancer + routing configuration generated from HTTPRoutes + MUST prioritize rules based on the following + criteria, continuing on ties. Precedence must + be given to the the Rule with the largest + number of: \n * Characters in a matching non-wildcard + hostname. * Characters in a matching hostname. + * Characters in a matching path. * Header + matches. * Query param matches. \n If ties + still exist across multiple Routes, matching + precedence MUST be determined in order of + the following criteria, continuing on ties: + \n * The oldest Route based on creation timestamp. + * The Route appearing first in alphabetical + order by \"/\". \n If ties + still exist within the Route that has been + given precedence, matching precedence MUST + be granted to the first matching rule meeting + the above criteria." + items: + description: "HTTPRouteMatch defines the predicate + used to match requests to a given action. + Multiple match types are ANDed together, + i.e. the match will evaluate to true only + if all conditions are satisfied. \n For + example, the match below will match a HTTP + request only if its path starts with `/foo` + AND it contains the `version: v1` header: + \n ``` match: path: value: \"/foo\" + \ headers: - name: \"version\" value + \"v1\" ```" + properties: + headers: + description: Headers specifies HTTP request + header matchers. Multiple match values + are ANDed together, meaning, a request + must match all the specified headers + to select the route. + items: + description: HTTPHeaderMatch describes + how to select a HTTP route by matching + HTTP request headers. + properties: + name: + description: "Name is the name of + the HTTP Header to be matched. + Name matching MUST be case insensitive. + (See https://tools.ietf.org/html/rfc7230#section-3.2). + \n If multiple entries specify + equivalent header names, only + the first entry with an equivalent + name MUST be considered for a + match. Subsequent entries with + an equivalent header name MUST + be ignored. Due to the case-insensitivity + of header names, \"foo\" and \"Foo\" + are considered equivalent. \n + When a header is repeated in an + HTTP request, it is implementation-specific + behavior as to how this is represented. + Generally, proxies should follow + the guidance from the RFC: https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 + regarding processing a repeated + header, with special handling + for \"Set-Cookie\"." + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + type: + default: Exact + description: "Type specifies how + to match against the value of + the header. \n Support: Core (Exact) + \n Support: Custom (RegularExpression) + \n Since RegularExpression HeaderMatchType + has custom conformance, implementations + can support POSIX, PCRE or any + other dialects of regular expressions. + Please read the implementation's + documentation to determine the + supported dialect." + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value + of HTTP Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + method: + description: "Method specifies HTTP method + matcher. When specified, this route + will be matched only if the request + has the specified method. \n Support: + Extended" + enum: + - GET + - HEAD + - POST + - PUT + - DELETE + - CONNECT + - OPTIONS + - TRACE + - PATCH + type: string + path: + default: + type: PathPrefix + value: / + description: Path specifies a HTTP request + path matcher. If this field is not specified, + a default prefix match on the "/" path + is provided. + properties: + type: + default: PathPrefix + description: "Type specifies how to + match against the path Value. \n + Support: Core (Exact, PathPrefix) + \n Support: Custom (RegularExpression)" + enum: + - Exact + - PathPrefix + - RegularExpression + type: string + value: + default: / + description: Value of the HTTP path + to match against. + maxLength: 1024 + type: string + type: object + queryParams: + description: QueryParams specifies HTTP + query parameter matchers. Multiple match + values are ANDed together, meaning, + a request must match all the specified + query parameters to select the route. + items: + description: HTTPQueryParamMatch describes + how to select a HTTP route by matching + HTTP query parameters. + properties: + name: + description: Name is the name of + the HTTP query param to be matched. + This must be an exact string match. + (See https://tools.ietf.org/html/rfc7230#section-2.7.3). + maxLength: 256 + minLength: 1 + type: string + type: + default: Exact + description: "Type specifies how + to match against the value of + the query parameter. \n Support: + Extended (Exact) \n Support: Custom + (RegularExpression) \n Since RegularExpression + QueryParamMatchType has custom + conformance, implementations can + support POSIX, PCRE or any other + dialects of regular expressions. + Please read the implementation's + documentation to determine the + supported dialect." + enum: + - Exact + - RegularExpression + type: string + value: + description: Value is the value + of HTTP query param to be matched. + maxLength: 1024 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + maxItems: 8 + type: array + type: object + maxItems: 16 + type: array + type: object + status: + description: Status defines the current state of HTTPRoute. + properties: + parents: + description: "Parents is a list of parent resources + (usually Gateways) that are associated with the + route, and the status of the route with respect + to each parent. When this route attaches to a parent, + the controller that manages the parent must add + an entry to this list when the controller first + sees the route and should update the entry as appropriate + when the route or gateway is modified. \n Note that + parent references that cannot be resolved by an + implementation of this API will not be added to + this list. Implementations of this API can only + populate Route status for the Gateways/parent resources + they are responsible for. \n A maximum of 32 Gateways + will be represented in this list. An empty list + means the route has not been attached to any Gateway." + items: + description: RouteParentStatus describes the status + of a route with respect to an associated Parent. + properties: + conditions: + description: "Conditions describes the status + of the route with respect to the Gateway. + Note that the route's availability is also + subject to the Gateway's own status conditions + and listener status. \n If the Route's ParentRef + specifies an existing Gateway that supports + Routes of this kind AND that Gateway's controller + has sufficient access, then that Gateway's + controller MUST set the \"Accepted\" condition + on the Route, to indicate whether the route + has been accepted or rejected by the Gateway, + and why. \n A Route MUST be considered \"Accepted\" + if at least one of the Route's rules is implemented + by the Gateway. \n There are a number of cases + where the \"Accepted\" condition may not be + set due to lack of controller visibility, + that includes when: \n * The Route refers + to a non-existent parent. * The Route is of + a type that the controller does not support. + * The Route is in a namespace the the controller + does not have access to." + items: + description: "Condition contains details for + one aspect of the current state of this + API Resource. --- This struct is intended + for direct use as an array at the field + path .status.conditions. For example, type + FooStatus struct{ // Represents the + observations of a foo's current state. // + Known .status.conditions.type are: \"Available\", + \"Progressing\", and \"Degraded\" // + +patchMergeKey=type // +patchStrategy=merge + \ // +listType=map // +listMapKey=type + \ Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" + protobuf:\"bytes,1,rep,name=conditions\"` + \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the + last time the condition transitioned + from one status to another. This should + be when the underlying condition changed. If + that is not known, then using the time + when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable + message indicating details about the + transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents + the .metadata.generation that the condition + was set based upon. For instance, if + .metadata.generation is currently 12, + but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with + respect to the current state of the + instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic + identifier indicating the reason for + the condition's last transition. Producers + of specific condition types may define + expected values and meanings for this + field, and whether the values are considered + a guaranteed API. The value should be + a CamelCase string. This field may not + be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, + one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase + or in foo.example.com/CamelCase. --- + Many .condition.type values are consistent + across resources like Available, but + because arbitrary conditions can be + useful (see .node.status.conditions), + the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controllerName: + description: "ControllerName is a domain/path + string that indicates the name of the controller + that wrote this status. This corresponds with + the controllerName field on GatewayClass. + \n Example: \"example.net/gateway-controller\". + \n The format of this field is DOMAIN \"/\" + PATH, where DOMAIN and PATH are valid Kubernetes + names (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names)." + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + parentRef: + description: ParentRef corresponds with a ParentRef + in the spec that this RouteParentStatus struct + describes the status of. + properties: + group: + default: gateway.networking.k8s.io + description: "Group is the group of the + referent. \n Support: Core" + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: "Kind is kind of the referent. + \n Support: Core (Gateway) Support: Custom + (Other Resources)" + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: "Name is the name of the referent. + \n Support: Core" + maxLength: 253 + minLength: 1 + type: string + namespace: + description: "Namespace is the namespace + of the referent. When unspecified (or + empty string), this refers to the local + namespace of the Route. \n Support: Core" + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + sectionName: + description: "SectionName is the name of + a section within the target resource. + In the following resources, SectionName + is interpreted as the following: \n * + Gateway: Listener Name \n Implementations + MAY choose to support attaching Routes + to other resources. If that is the case, + they MUST clearly document how SectionName + is interpreted. \n When unspecified (empty + string), this will reference the entire + resource. For the purpose of status, an + attachment is considered successful if + at least one section in the parent resource + accepts it. For example, Gateway listeners + can restrict which Routes can attach to + them by Route kind, namespace, or hostname. + If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route + MUST be considered successfully attached. + If no Gateway listeners accept attachment + from this Route, the Route MUST be considered + detached from the Gateway. \n Support: + Core" + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + required: + - controllerName + - parentRef + type: object + maxItems: 32 + type: array + required: + - parents + type: object + required: + - spec + type: object + name: + description: Name indicates the httproute name + type: string + type: object + ingressWrapper: + description: Ingress indicates information of ingress + properties: + ingress: + description: Ingress indicates the ingress + properties: + apiVersion: + description: 'APIVersion defines the versioned schema + of this representation of an object. Servers should + convert recognized schemas to the latest internal value, + and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the + REST resource this object represents. Servers may infer + this from the endpoint the client submits requests to. + Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: 'Spec is the desired state of the Ingress. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + defaultBackend: + description: DefaultBackend is the backend that should + handle requests that don't match any rule. If Rules + are not specified, DefaultBackend must be specified. + If DefaultBackend is not set, the handling of requests + that do not match any of the rules will be up to + the Ingress controller. + properties: + resource: + description: Resource is an ObjectRef to another + Kubernetes resource in the namespace of the + Ingress object. If resource is specified, a + service.Name and service.Port must not be specified. + This is a mutually exclusive setting with "Service". + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + service: + description: Service references a Service as a + Backend. This is a mutually exclusive setting + with "Resource". + properties: + name: + description: Name is the referenced service. + The service must exist in the same namespace + as the Ingress object. + type: string + port: + description: Port of the referenced service. + A port name or port number is required for + a IngressServiceBackend. + properties: + name: + description: Name is the name of the port + on the Service. This is a mutually exclusive + setting with "Number". + type: string + number: + description: Number is the numerical port + number (e.g. 80) on the Service. This + is a mutually exclusive setting with + "Name". + format: int32 + type: integer + type: object + required: + - name + type: object + type: object + ingressClassName: + description: IngressClassName is the name of the IngressClass + cluster resource. The associated IngressClass defines + which controller will implement the resource. This + replaces the deprecated `kubernetes.io/ingress.class` + annotation. For backwards compatibility, when that + annotation is set, it must be given precedence over + this field. The controller may emit a warning if + the field and annotation have different values. + Implementations of this API should ignore Ingresses + without a class specified. An IngressClass resource + may be marked as default, which can be used to set + a default value for this field. For more information, + refer to the IngressClass documentation. + type: string + rules: + description: A list of host rules used to configure + the Ingress. If unspecified, or no rule matches, + all traffic is sent to the default backend. + items: + description: IngressRule represents the rules mapping + the paths under a specified host to the related + backend services. Incoming requests are first + evaluated for a host match, then routed to the + backend associated with the matching IngressRuleValue. + properties: + host: + description: "Host is the fully qualified domain + name of a network host, as defined by RFC + 3986. Note the following deviations from the + \"host\" part of the URI as defined in RFC + 3986: 1. IPs are not allowed. Currently an + IngressRuleValue can only apply to the + IP in the Spec of the parent Ingress. 2. The + `:` delimiter is not respected because ports + are not allowed. \t Currently the port of + an Ingress is implicitly :80 for http and + \t :443 for https. Both these may change + in the future. Incoming requests are matched + against the host before the IngressRuleValue. + If the host is unspecified, the Ingress routes + all traffic based on the specified IngressRuleValue. + \n Host can be \"precise\" which is a domain + name without the terminating dot of a network + host (e.g. \"foo.bar.com\") or \"wildcard\", + which is a domain name prefixed with a single + wildcard label (e.g. \"*.foo.com\"). The wildcard + character '*' must appear by itself as the + first DNS label and matches only a single + label. You cannot have a wildcard label by + itself (e.g. Host == \"*\"). Requests will + be matched against the Host field in the following + way: 1. If Host is precise, the request matches + this rule if the http host header is equal + to Host. 2. If Host is a wildcard, then the + request matches this rule if the http host + header is to equal to the suffix (removing + the first label) of the wildcard rule." + type: string + http: + description: 'HTTPIngressRuleValue is a list + of http selectors pointing to backends. In + the example: http:///? + -> backend where where parts of the url correspond + to RFC 3986, this resource will be used to + match against everything after the last ''/'' + and before the first ''?'' or ''#''.' + properties: + paths: + description: A collection of paths that + map requests to backends. + items: + description: HTTPIngressPath associates + a path with a backend. Incoming urls + matching the path are forwarded to the + backend. + properties: + backend: + description: Backend defines the referenced + service endpoint to which the traffic + will be forwarded to. + properties: + resource: + description: Resource is an ObjectRef + to another Kubernetes resource + in the namespace of the Ingress + object. If resource is specified, + a service.Name and service.Port + must not be specified. This + is a mutually exclusive setting + with "Service". + properties: + apiGroup: + description: APIGroup is the + group for the resource being + referenced. If APIGroup + is not specified, the specified + Kind must be in the core + API group. For any other + third-party types, APIGroup + is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + service: + description: Service references + a Service as a Backend. This + is a mutually exclusive setting + with "Resource". + properties: + name: + description: Name is the referenced + service. The service must + exist in the same namespace + as the Ingress object. + type: string + port: + description: Port of the referenced + service. A port name or + port number is required + for a IngressServiceBackend. + properties: + name: + description: Name is the + name of the port on + the Service. This is + a mutually exclusive + setting with "Number". + type: string + number: + description: Number is + the numerical port number + (e.g. 80) on the Service. + This is a mutually exclusive + setting with "Name". + format: int32 + type: integer + type: object + required: + - name + type: object + type: object + path: + description: Path is matched against + the path of an incoming request. + Currently it can contain characters + disallowed from the conventional + "path" part of a URL as defined + by RFC 3986. Paths must begin with + a '/' and must be present when using + PathType with value "Exact" or "Prefix". + type: string + pathType: + description: 'PathType determines + the interpretation of the Path matching. + PathType can be one of the following + values: * Exact: Matches the URL + path exactly. * Prefix: Matches + based on a URL path prefix split + by ''/''. Matching is done on + a path element by element basis. + A path element refers is the list + of labels in the path split by the + ''/'' separator. A request is a match + for path p if every p is an element-wise + prefix of p of the request path. + Note that if the last element of + the path is a substring of the + last element in request path, it + is not a match (e.g. /foo/bar matches + /foo/bar/baz, but does not match + /foo/barbaz). * ImplementationSpecific: + Interpretation of the Path matching + is up to the IngressClass. Implementations + can treat this as a separate PathType or + treat it identically to Prefix or + Exact path types. Implementations + are required to support all path + types.' + type: string + required: + - backend + - pathType + type: object + type: array + x-kubernetes-list-type: atomic + required: + - paths + type: object + type: object + type: array + x-kubernetes-list-type: atomic + tls: + description: TLS configuration. Currently the Ingress + only supports a single TLS port, 443. If multiple + members of this list specify different hosts, they + will be multiplexed on the same port according to + the hostname specified through the SNI TLS extension, + if the ingress controller fulfilling the ingress + supports SNI. + items: + description: IngressTLS describes the transport + layer security associated with an Ingress. + properties: + hosts: + description: Hosts are a list of hosts included + in the TLS certificate. The values in this + list must match the name/s used in the tlsSecret. + Defaults to the wildcard host setting for + the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: SecretName is the name of the secret + used to terminate TLS traffic on port 443. + Field is left optional to allow TLS routing + based on SNI hostname alone. If the SNI host + in a listener conflicts with the "Host" header + field used by an IngressRule, the SNI host + is used for termination and value of the Host + header is used for routing. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + type: object + status: + description: 'Status is the current state of the Ingress. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + loadBalancer: + description: LoadBalancer contains the current status + of the load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress + points for the load-balancer. Traffic intended + for the service should be sent to these ingress + points. + items: + description: 'LoadBalancerIngress represents + the status of a load-balancer ingress point: + traffic intended for the service should be + sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer + ingress points that are DNS based (typically + AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer + ingress points that are IP based (typically + GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records + of service ports If used, every port defined + in the service should have an entry in + it + items: + properties: + error: + description: 'Error is to record the + problem with the service port The + format of the error shall comply + with the following rules: - built-in + error values shall be specified + in this file and those shall use CamelCase + names - cloud provider specific + error values must have names that + comply with the format foo.example.com/CamelCase. + --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number + of the service port of which status + is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol + of the service port of which status + is recorded here The supported values + are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + type: object + type: object + name: + description: Name indicates the ingress name + type: string + type: object + type: object + workload: + description: Workload indicates information of the workload, such + as cloneset, deployment, advanced statefulset + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name indicates the workload name + type: string + replicas: + description: Number of desired pods. This is a pointer to distinguish + between explicit zero and not specified. Defaults to 1. + format: int32 + type: integer + selector: + description: Label selector for pods. It must match the pod template's + labels. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + template: + description: Template describes the pods that will be created. + x-kubernetes-preserve-unknown-fields: true + type: + description: Type of deployment or CloneSetUpdateStrategy. Can + be "Recreate" or "RollingUpdate". + type: string + required: + - selector + - template + type: object + type: object + status: + description: RolloutHistoryStatus defines the observed state of RolloutHistory + properties: + canaryStepIndex: + description: CanaryStepIndex indicates the current step + format: int32 + type: integer + canaryStepPods: + description: canaryStepPods indicates the pods released + items: + description: CanaryStepPods indicates the pods for a revision + properties: + pods: + description: Pods indicates the pods information + items: + description: Pod indicates the information of a pod, including + name, ip, node_name. + properties: + ip: + description: IP indicates the pod ip + type: string + name: + description: Name indicates the node name + type: string + node: + description: Node indicates the node which pod is located + at + type: string + type: object + type: array + podsInStep: + description: PodsInStep indicates the num of new pods released + this step + format: int32 + type: integer + podsInTotal: + description: PodsInTotal indicates the num of new pods released + by now + format: int32 + type: integer + stepIndex: + description: StepIndex indicates the step index + format: int32 + type: integer + required: + - podsInStep + - podsInTotal + type: object + type: array + canaryStepState: + description: CanaryStepState indicates state of this rollout revision, + such as "init", "pending", "update", "terminated", "completed", + "cancelled", whick is upon rollout canary_step_state + type: string + phase: + description: Phase indicates phase of RolloutHistory, such as "pending", + "progressing", "completed" + type: string + rolloutState: + description: RolloutState indicates the rollouts status + properties: + message: + description: Message provides details on why the rollout is in + its current phase + type: string + rolloutPhase: + description: RolloutPhase is the rollout phase. + type: string + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index a621998..d50dc06 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/rollouts.kruise.io_rollouts.yaml - bases/rollouts.kruise.io_batchreleases.yaml +- bases/rollouts.kruise.io_rollouthistories.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -11,14 +12,16 @@ patchesStrategicMerge: # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_rollouts.yaml #- patches/webhook_in_batchreleases.yaml +#- patches/webhook_in_rollouthistories.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_rollouts.yaml #- patches/cainjection_in_batchreleases.yaml +#- patches/cainjection_in_rollouthistories.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. configurations: -- kustomizeconfig.yaml +- kustomizeconfig.yaml \ No newline at end of file diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 97cb66a..cc69d72 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -262,6 +262,32 @@ rules: - get - patch - update +- apiGroups: + - rollouts.kruise.io + resources: + - rollouthistories + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rollouts.kruise.io + resources: + - rollouthistories/finalizers + verbs: + - update +- apiGroups: + - rollouts.kruise.io + resources: + - rollouthistories/status + verbs: + - get + - patch + - update - apiGroups: - rollouts.kruise.io resources: diff --git a/main.go b/main.go index 85a720c..31e4824 100644 --- a/main.go +++ b/main.go @@ -22,12 +22,6 @@ import ( kruisev1aplphal1 "github.com/openkruise/kruise-api/apps/v1alpha1" kruisev1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" - rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" - br "github.com/openkruise/rollouts/pkg/controller/batchrelease" - "github.com/openkruise/rollouts/pkg/controller/rollout" - "github.com/openkruise/rollouts/pkg/util" - utilclient "github.com/openkruise/rollouts/pkg/util/client" - "github.com/openkruise/rollouts/pkg/webhook" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -37,6 +31,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + br "github.com/openkruise/rollouts/pkg/controller/batchrelease" + "github.com/openkruise/rollouts/pkg/controller/rollout" + "github.com/openkruise/rollouts/pkg/controller/rollouthistory" + "github.com/openkruise/rollouts/pkg/util" + utilclient "github.com/openkruise/rollouts/pkg/util/client" + "github.com/openkruise/rollouts/pkg/webhook" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -109,6 +111,14 @@ func main() { os.Exit(1) } + if err = (&rollouthistory.RolloutHistoryReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("rollouthistory-controller"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "RolloutHistory") + os.Exit(1) + } //+kubebuilder:scaffold:builder setupLog.Info("setup webhook") if err = webhook.SetupWithManager(mgr); err != nil { diff --git a/pkg/controller/rollouthistory/rollouthistory_controller.go b/pkg/controller/rollouthistory/rollouthistory_controller.go new file mode 100644 index 0000000..6e16522 --- /dev/null +++ b/pkg/controller/rollouthistory/rollouthistory_controller.go @@ -0,0 +1,100 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollouthistory + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" +) + +// RolloutHistoryReconciler reconciles a RolloutHistory object +type RolloutHistoryReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder +} + +//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=rollouthistories,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=rollouthistories/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=rollouthistories/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the RolloutHistory object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.10.0/pkg/reconcile +func (r *RolloutHistoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + + // get rollout + rollout := &rolloutv1alpha1.Rollout{} + err := r.Get(ctx, req.NamespacedName, rollout) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // get RolloutHistory which is not completed and related to the rollout (only one or zero) + var rollouthistory *rolloutv1alpha1.RolloutHistory + if rollouthistory, err = r.getRHByRollout(rollout); err != nil { + // if not find RolloutHistory related this rollout, skip some situations + if rollout.Status.CanaryStatus == nil || rollout.Spec.Strategy.Canary == nil || rollout.Spec.ObjectRef.WorkloadRef == nil || rollout.Spec.RolloutID == "" || rollout.Status.CanaryStatus.CurrentStepIndex != 0 { + return ctrl.Result{}, nil + } + // create a rollouthistory when user do a new rollout + if rollouthistory, err = r.createRHByRollout(rollout); err != nil { + return ctrl.Result{}, err + } + } + + // handle rollouthistory according to rollouthistory.status.canaryStepState + switch rollouthistory.Status.CanaryStepState { + case rolloutv1alpha1.CanaryStateInit: + err = r.handleInit(rollout, rollouthistory) + case rolloutv1alpha1.CanaryStatePending: + err = r.handlePending(rollout, rollouthistory) + case rolloutv1alpha1.CanaryStateUpdated: + err = r.handleUpdated(rollout, rollouthistory) + case rolloutv1alpha1.CanaryStateTerminated: + err = r.handleTerminated(rollout, rollouthistory) + case rolloutv1alpha1.CanaryStateCompleted: + err = r.handleCompleted(rollout, rollouthistory) + } + + if err != nil { + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *RolloutHistoryReconciler) SetupWithManager(mgr ctrl.Manager) error { + + return ctrl.NewControllerManagedBy(mgr). + For(&rolloutv1alpha1.Rollout{}). + Complete(r) + +} diff --git a/pkg/controller/rollouthistory/rollouthistory_event_handler.go b/pkg/controller/rollouthistory/rollouthistory_event_handler.go new file mode 100644 index 0000000..1ca7191 --- /dev/null +++ b/pkg/controller/rollouthistory/rollouthistory_event_handler.go @@ -0,0 +1,669 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollouthistory + +import ( + "context" + "errors" + "fmt" + "sort" + + kruiseapi "github.com/openkruise/kruise-api/apps/v1alpha1" + kruiseapi_beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + v1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" +) + +// get rollouthistory according to rollout. The rollouthistory should be not completed now +func (r *RolloutHistoryReconciler) getRHByRollout(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.RolloutHistory, error) { + // get all rollouthistories + rhList := &rolloutv1alpha1.RolloutHistoryList{} + err := r.List(context.TODO(), rhList, &client.ListOptions{}, client.InNamespace(rollout.Namespace)) + if err != nil { + return nil, err + } + + // if the num of rollouthistory is greater than MaxRolloutHistoryNum, it will delete excess rollouthistories. + // sort rollouthistories by creationTimeStamp + result := rhList.Items + sort.Slice(result, func(i, j int) bool { + return result[i].CreationTimestamp.Before(&result[j].CreationTimestamp) + }) + + lenth := len(result) + if lenth > rolloutv1alpha1.MaxRolloutHistoryNum { + for _, rollouthistory := range result[:lenth-rolloutv1alpha1.MaxRolloutHistoryNum] { + thisRolloutHistory := rollouthistory + err = r.Delete(context.TODO(), &thisRolloutHistory, &client.DeleteOptions{}) + if err != nil { + return nil, err + } + } + result = append(result[:0], result[lenth-rolloutv1alpha1.MaxRolloutHistoryNum:]...) + } + + // get target rollouthistory according to rollout.Spec.RolloutID and rollout.Name + for i := range result { + RH := result[i] + if rollout.Spec.RolloutID == RH.Spec.RolloutWrapper.Rollout.Spec.RolloutID && + rollout.Name == RH.Spec.RolloutWrapper.Name && + RH.Status.Phase != rolloutv1alpha1.PhaseCompleted { + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.getRHByRollout successed", " ") + return &RH, nil + } + } + + return nil, errors.New("getRHByRollout can't find RH") +} + +// create a rollouthistory which happens when user do a rollout +func (r *RolloutHistoryReconciler) createRHByRollout(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.RolloutHistory, error) { + logger := log.FromContext(context.TODO()) + // init .spec.rolloutWrapper + rolloutWrapper, err := r.initRolloutWrapper(rollout) + if err != nil { + logger.Info(err.Error() + " after r.initRolloutInfo") + return nil, err + } + + // init .spec.workload + workload, err := r.initWorkloadInfo(rollout) + if err != nil { + logger.Info(err.Error() + " after r.initWorkloadInfo") + return nil, err + } + + // init .spec.serviceWrapper + serviceWrapper, err := r.initServiceWrapper(rollout, workload) + if err != nil { + logger.Info(err.Error() + " after r.initServiceInfo") + return nil, err + } + + // init .spec.traffcRoutingWrapper + trafficRoutingWrapper, err := r.initTrafficRoutingWrapper(rollout) + if err != nil { + logger.Info(err.Error() + " after r.initIngressInfo") + return nil, err + } + + // init rollouthistory + RH := &rolloutv1alpha1.RolloutHistory{ + TypeMeta: v1.TypeMeta{ + Kind: "RolloutHistory", + APIVersion: "rollouts.kruise.io/v1alpha1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: rollout.Name + "-rh-" + RandAllString(6), + Namespace: rollout.Namespace, + }, + Spec: rolloutv1alpha1.RolloutHistorySpec{ + RolloutWrapper: *rolloutWrapper, + Workload: *workload, + ServiceWrapper: *serviceWrapper, + TrafficRoutingWrapper: *trafficRoutingWrapper, + }, + Status: rolloutv1alpha1.RolloutHistoryStatus{}, + } + + // create this rollouthistory + err = r.Create(context.TODO(), RH, &client.CreateOptions{}) + if err != nil { + logger.Info(err.Error() + " after r.Create") + return nil, err + } + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.createRHByRollout.1 successed", RH.Status.RolloutState.Message) + + // init .status for rollouthistory + if err = r.initRHStatus(rollout, RH); err != nil { + logger.Info(err.Error() + " after r.initRHStatus") + return nil, err + } + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.createRHByRollout.2 successed", RH.Status.RolloutState.Message) + + return RH, nil +} + +// handle function, when rollouthistory .status.canaryStepState is CanaryStateInit +func (r *RolloutHistoryReconciler) handleInit(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + // when rollouthistory is meet the need in CanaryStateInit, move it to next canaryStepState + if *RH.Status.CanaryStepIndex == 0 && + RT.Status.CanaryStatus.CurrentStepState == rolloutv1alpha1.CanaryStepStateUpgrade && + *RH.Status.CanaryStepIndex+1 == RT.Status.CanaryStatus.CurrentStepIndex { + RH.Status.CanaryStepState = rolloutv1alpha1.CanaryStatePending + *RH.Status.CanaryStepIndex += 1 + RH.Status.Phase = rolloutv1alpha1.PhaseProgressing + + // update this rollouthistory + err := r.Status().Update(context.TODO(), RH, &client.UpdateOptions{}) + if err != nil { + return err + } + + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.handleInit successed", RH.Status.RolloutState.Message) + } + return nil +} + +// handle function, when rollouthistory .status.canaryStepState is CanaryStatePending +func (r *RolloutHistoryReconciler) handlePending(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + // if rollouthistory .status.canaryStepIndex is equal to rollout .status.canaryStatus.currentStepIndex, and + // rollout .status.canaryStatus.currentStepState is CanaryStepStatePaused, rollouthistory need to record pod released information + // in this canary step + if RT.Status.CanaryStatus.CurrentStepIndex == int32(*RH.Status.CanaryStepIndex) && + RT.Status.CanaryStatus.CurrentStepState == rolloutv1alpha1.CanaryStepStatePaused { + // rollout is in StepPaused, update info + var err error + // update .status.rolloutState + err = r.updateStatusRolloutState(RT, RH) + if err != nil { + return err + } + // update .status.canaryStepPods information in this step + err = r.updateCanaryStepPods(RT, RH) + if err != nil { + return err + } + + // when update done, rollouthistory .status.canaryStepState should be CanaryStateUpdated + RH.Status.CanaryStepState = rolloutv1alpha1.CanaryStateUpdated + + err = r.Status().Update(context.TODO(), RH, &client.UpdateOptions{}) + if err != nil { + return err + } + + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.handlePending successed", RH.Status.RolloutState.Message) + } + + // if rollout .status.canaryStatus.currentStepIndex is 0 now, it means that user do a continuous rollout v1 -> v2(not completed) -> v3 + if RT.Status.CanaryStatus.CurrentStepIndex == 0 { + err := r.doTerminated(RT, RH) + if err != nil { + return err + } + + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.doTerminated successed", RH.Name) + } + + // if rollout .status.phase is RolloutPhaseHealthy now, it means that user do a continuous rollout v1 -> v2(not completed) -> v1(back) + if RT.Status.Phase == rolloutv1alpha1.RolloutPhaseHealthy && + RT.Status.CanaryStatus.CurrentStepState != rolloutv1alpha1.CanaryStepStateCompleted { + err := r.doCancelled(RT, RH) + if err != nil { + return err + } + + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.doCancelled successed", RH.Name) + } + + return nil +} + +// handle function, when rollouthistory .status.canaryStepState is CanaryStateUpdated +func (r *RolloutHistoryReconciler) handleUpdated(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + // if rollout .status.canaryStatus.currentStepState is canaryStepStateUpgrade, it means that the last canary step has been approved + if RT.Status.CanaryStatus.CurrentStepState == rolloutv1alpha1.CanaryStepStateUpgrade && + RT.Status.CanaryStatus.CurrentStepIndex == int32(*RH.Status.CanaryStepIndex)+1 { + RH.Status.CanaryStepState = rolloutv1alpha1.CanaryStatePending + *RH.Status.CanaryStepIndex++ + + err := r.Status().Update(context.TODO(), RH, &client.UpdateOptions{}) + if err != nil { + return err + } + + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.handleUpdate successed", RH.Status.RolloutState.Message) + } + + // if rollout .status.canaryStatus.currentStepState is CanaryStepStateCompleted now , it means that this rollout has completed + if RT.Status.CanaryStatus.CurrentStepState == rolloutv1alpha1.CanaryStepStateCompleted && + RT.Status.CanaryStatus.CurrentStepIndex == int32(*RH.Status.CanaryStepIndex) { + err := r.doCompleted(RT, RH) + if err != nil { + return err + } + + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.doCompleted successed", RH.Status.RolloutState.Message) + } + + // if rollout .status.phase is RolloutPhaseHealthy now, it means that user do a continuous rollout v1 -> v2(not completed) -> v1(back) + if RT.Status.Phase == rolloutv1alpha1.RolloutPhaseHealthy && + RT.Status.CanaryStatus.CurrentStepState != rolloutv1alpha1.CanaryStepStateCompleted { + err := r.doCancelled(RT, RH) + if err != nil { + return err + } + + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.doCancelled successed", RH.Name) + } + + // if rollout .status.canaryStatus.currentStepIndex is 0 now, it means that user do a continuous rollout v1 -> v2(not completed) -> v3 + if RT.Status.CanaryStatus.CurrentStepIndex == 0 { + err := r.doTerminated(RT, RH) + if err != nil { + return err + } + + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.doTerminated successed", RH.Name) + } + + return nil +} + +// handle function, when rollouthistory .status.canaryStepState is CanaryStateTerminated, when user do a continuous rollout v1 -> v2(not completed) -> v3 +func (r *RolloutHistoryReconciler) handleTerminated(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + if RH.Status.Phase == rolloutv1alpha1.PhaseCompleted { + return nil + } + + // update this terminated-rollout related rollouthistory(v2 not completed), make .status.phase completed + RH.Status.Phase = rolloutv1alpha1.PhaseCompleted + err := r.Status().Update(context.TODO(), RH, &client.UpdateOptions{}) + if err != nil { + return err + } + + // when user do a continuous rollout v1 -> v2(not completed) -> v3, it will create a new rollouthistory for this rollout + newWorkload, err := r.initWorkloadInfo(RT) + if err != nil { + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.handleTerminated and initWorKload failed", RH.Status.RolloutState.Message) + return err + } + + // get this new rollout which rolloutID is different and rollout name is same + newRollout := &rolloutv1alpha1.Rollout{} + err = r.Get(context.TODO(), types.NamespacedName{Namespace: RT.Namespace, Name: RT.Name}, newRollout) + if err != nil { + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.handleTerminated and get newRollout failed", RH.Status.RolloutState.Message) + return err + } + + // init rolloutWrapper for the new rollouthistory + newRolloutWrapper, err := r.initRolloutWrapper(newRollout) + if err != nil { + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.handleTerminated and initRolloutInfo failed", RH.Status.RolloutState.Message) + return err + } + + // init this new rollouthistory for rollout(v3) + newRolloutHistory := &rolloutv1alpha1.RolloutHistory{ + TypeMeta: v1.TypeMeta{ + Kind: RH.Kind, + APIVersion: RH.APIVersion, + }, + ObjectMeta: v1.ObjectMeta{ + Name: RT.Name + "-rh-" + RandAllString(6), + Namespace: RT.Namespace, + }, + Spec: rolloutv1alpha1.RolloutHistorySpec{ + RolloutWrapper: *newRolloutWrapper, + Workload: *newWorkload, + TrafficRoutingWrapper: RH.Spec.TrafficRoutingWrapper, + ServiceWrapper: RH.Spec.ServiceWrapper, + }, + Status: rolloutv1alpha1.RolloutHistoryStatus{}, + } + + // create this rollouthistory, which record information of rollout(v3) + err = r.Create(context.TODO(), newRolloutHistory, &client.CreateOptions{}) + if err != nil { + return err + } + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.createRHByRollout.1 successed", RH.Status.RolloutState.Message) + + // init .status for rollouthistory + if err = r.initRHStatus(RT, newRolloutHistory); err != nil { + return err + } + + r.Recorder.Event(newRolloutHistory.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.createRHByRollout.2 successed", RH.Status.RolloutState.Message) + + return nil +} + +// handle function, when rollouthistory .status.canaryStepState is CanaryStateCompleted +func (r *RolloutHistoryReconciler) handleCompleted(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + if RH.Status.Phase != rolloutv1alpha1.PhaseCompleted && RT.Status.Phase == rolloutv1alpha1.RolloutPhaseHealthy { + var err error + err = r.updateStatusRolloutState(RT, RH) + if err != nil { + return err + } + + // update the final step status + if *RT.Spec.Strategy.Canary.Steps[*RH.Status.CanaryStepIndex-1].Weight != 100 { + *RH.Status.CanaryStepIndex += 1 + err = r.updateCanaryStepPods(RT, RH) + if err != nil { + return err + } + } + + RH.Status.Phase = rolloutv1alpha1.PhaseCompleted + err = r.Status().Update(context.TODO(), RH, &client.UpdateOptions{}) + if err != nil { + return err + } + } + return nil +} + +// handle function when rollout process is terminated, which means that user do a new rollout +func (r *RolloutHistoryReconciler) doTerminated(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + // just make .status.canaryStepState CanaryStateTerminated + RH.Status.CanaryStepState = rolloutv1alpha1.CanaryStateTerminated + err := r.Status().Update(context.TODO(), RH, &client.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} + +// handle function when rollout process is cancelled, which means that user do a rollback +func (r *RolloutHistoryReconciler) doCancelled(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + // if rollout is cancelled, make rollouthistory .status.canaryStepState CanaryStateCompleted and .status.phase PhaseCompleted + RH.Status.CanaryStepState = rolloutv1alpha1.CanaryStateCompleted + RH.Status.Phase = rolloutv1alpha1.PhaseCompleted + err := r.updateStatusRolloutState(RT, RH) + if err != nil { + return err + } + + // if rollout is cancelled, make rollouthistory .status.canaryStepIndex value "-1" + *RH.Status.CanaryStepIndex = -1 + err = r.Status().Update(context.TODO(), RH, &client.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} + +// handle function doComplated, usually it be called when something unexpected happens and rollout .status.canaryStatus.currentStepState become CanaryStateCompleted +func (r *RolloutHistoryReconciler) doCompleted(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + RH.Status.CanaryStepState = rolloutv1alpha1.CanaryStateCompleted + + err := r.updateStatusRolloutState(RT, RH) + if err != nil { + return err + } + + err = r.Status().Update(context.TODO(), RH, &client.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} + +// init .status for rollouthistory +func (r *RolloutHistoryReconciler) initRHStatus(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + RH.Status.Phase = rolloutv1alpha1.PhaseInit + RH.Status.CanaryStepState = rolloutv1alpha1.CanaryStateInit + RH.Status.CanaryStepIndex = new(int32) + *RH.Status.CanaryStepIndex = 0 + RH.Status.RolloutState = rolloutv1alpha1.RolloutState{RolloutPhase: RT.Status.Phase, Message: RT.Status.Message} + RH.Status.CanaryStepPods = make([]rolloutv1alpha1.CanaryStepPods, 0) + + if err := r.Status().Update(context.TODO(), RH, &client.UpdateOptions{}); err != nil { + return err + } + r.Recorder.Event(RH.DeepCopy().DeepCopyObject(), corev1.EventTypeNormal, "r.initRHStatus succeed", " ") + return nil +} + +// init .spec.rolloutWrapper +func (r *RolloutHistoryReconciler) initRolloutWrapper(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.RolloutWrapper, error) { + RolloutWrapper := &rolloutv1alpha1.RolloutWrapper{ + Name: rollout.Name, + Rollout: *rollout, + } + return RolloutWrapper, nil +} + +// init .spec.traffcRoutingWrapper +func (r *RolloutHistoryReconciler) initTrafficRoutingWrapper(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.TrafficRoutingWrapper, error) { + trafficRoutingWrapper := &rolloutv1alpha1.TrafficRoutingWrapper{} + var err error + // if gateway is configured, init it + if rollout.Spec.Strategy.Canary.TrafficRoutings[0].Gateway != nil && + rollout.Spec.Strategy.Canary.TrafficRoutings[0].Gateway.HTTPRouteName != nil { + trafficRoutingWrapper.HTTPRouteWrapper, err = r.initGateWayInfo(rollout) + if err != nil { + return nil, err + } + } + + // if ingress is configured, init it + if rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress.Name != "" { + trafficRoutingWrapper.IngressWrapper, err = r.initIngressInfo(rollout) + if err != nil { + return nil, err + } + } + + return trafficRoutingWrapper, nil +} + +// init Gateway, especially HTTPRoute +func (r *RolloutHistoryReconciler) initGateWayInfo(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.HTTPRouteWrapper, error) { + GatewayName := *rollout.Spec.Strategy.Canary.TrafficRoutings[0].Gateway.HTTPRouteName + HTTPRoute := &v1alpha2.HTTPRoute{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: GatewayName}, HTTPRoute) + if err != nil { + return nil, errors.New("initGateway error: HTTPRoute " + GatewayName + " not find") + } + + return &rolloutv1alpha1.HTTPRouteWrapper{Name: GatewayName, HTTPRoute: HTTPRoute}, err +} + +// init Ingress +func (r *RolloutHistoryReconciler) initIngressInfo(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.IngressWrapper, error) { + IngressName := rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress.Name + Ingress := &networkingv1.Ingress{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: IngressName}, Ingress) + if err != nil { + return nil, errors.New("initIngressInfo error: Ingress " + IngressName + " not find") + } + + return &rolloutv1alpha1.IngressWrapper{Name: IngressName, Ingress: Ingress}, err +} + +// init .spec.workload +func (r *RolloutHistoryReconciler) initWorkloadInfo(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.Workload, error) { + // init specific workload, according to rollout.spec.objectRef.workloadRef.kind, now rollouthistory support Deployment, CloneSet, native StatefulSet and Advanced StatefulSet + switch rollout.Spec.ObjectRef.WorkloadRef.Kind { + case "Deployment": + return r.initDeploymentInfo(rollout) + case "CloneSet": + return r.initCloneSetInfo(rollout) + case "StatefulSet": + // According to rollout.spec.objectRef.workload.APIVersion, select Advanced StatefulSet or Native StatefulSet + if rollout.Spec.ObjectRef.WorkloadRef.APIVersion == "apps.kruise.io/v1beta1" { + return r.initAdvancedStatefulSetInfo(rollout) + } + if rollout.Spec.ObjectRef.WorkloadRef.APIVersion == "apps/v1" { + return r.initNativeStatefulSetInfo(rollout) + } + return nil, errors.New("StatefulSet is invalid") + default: + return nil, errors.New("workload is invalid") + } +} + +// init workload deployment +func (r *RolloutHistoryReconciler) initDeploymentInfo(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.Workload, error) { + deployment := &appsv1.Deployment{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: rollout.Spec.ObjectRef.WorkloadRef.Name}, deployment) + if err != nil { + return nil, errors.New("deployment not find") + } + + DeploymentInfo := rolloutv1alpha1.Workload{ + TypeMeta: deployment.TypeMeta, + Name: deployment.Name, + Selector: deployment.Spec.Selector, + Replicas: deployment.Spec.Replicas, + Type: string(deployment.Spec.Strategy.Type), + Template: deployment.Spec.Template, + } + + return &DeploymentInfo, nil +} + +// init workload cloneset +func (r *RolloutHistoryReconciler) initCloneSetInfo(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.Workload, error) { + cloneSet := &kruiseapi.CloneSet{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: rollout.Spec.ObjectRef.WorkloadRef.Name}, cloneSet) + if err != nil { + return &rolloutv1alpha1.Workload{}, err + } + + CloneSetInfo := rolloutv1alpha1.Workload{ + TypeMeta: cloneSet.TypeMeta, + Name: cloneSet.Name, + Selector: cloneSet.Spec.Selector, + Replicas: cloneSet.Spec.Replicas, + Type: string(cloneSet.Spec.UpdateStrategy.Type), + Template: cloneSet.Spec.Template, + } + + return &CloneSetInfo, nil +} + +// init Native StatefulSet +func (r *RolloutHistoryReconciler) initNativeStatefulSetInfo(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.Workload, error) { + cloneSet := &appsv1.StatefulSet{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: rollout.Spec.ObjectRef.WorkloadRef.Name}, cloneSet) + if err != nil { + return &rolloutv1alpha1.Workload{}, err + } + + CloneSetInfo := rolloutv1alpha1.Workload{ + TypeMeta: cloneSet.TypeMeta, + Name: cloneSet.Name, + Selector: cloneSet.Spec.Selector, + Replicas: cloneSet.Spec.Replicas, + Type: string(cloneSet.Spec.UpdateStrategy.Type), + Template: cloneSet.Spec.Template, + } + + return &CloneSetInfo, nil +} + +// init Advanced StatefulSet +func (r *RolloutHistoryReconciler) initAdvancedStatefulSetInfo(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.Workload, error) { + cloneSet := &kruiseapi_beta1.StatefulSet{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: rollout.Spec.ObjectRef.WorkloadRef.Name}, cloneSet) + if err != nil { + return &rolloutv1alpha1.Workload{}, err + } + + CloneSetInfo := rolloutv1alpha1.Workload{ + TypeMeta: cloneSet.TypeMeta, + Name: cloneSet.Name, + Selector: cloneSet.Spec.Selector, + Replicas: cloneSet.Spec.Replicas, + Type: string(cloneSet.Spec.UpdateStrategy.Type), + Template: cloneSet.Spec.Template, + } + + return &CloneSetInfo, nil +} + +// init .spec.serviceWrapper +func (r *RolloutHistoryReconciler) initServiceWrapper(rollout *rolloutv1alpha1.Rollout, workloadInfo *rolloutv1alpha1.Workload) (*rolloutv1alpha1.ServiceWrapper, error) { + Service := &corev1.Service{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service}, Service) + if err != nil { + return nil, errors.New("service not find") + } + + return &rolloutv1alpha1.ServiceWrapper{Name: Service.Name, Service: Service}, nil +} + +// update rollouthistory .status.rolloutState +func (r *RolloutHistoryReconciler) updateStatusRolloutState(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + RH.Status.RolloutState = rolloutv1alpha1.RolloutState{RolloutPhase: RT.Status.Phase, Message: RT.Status.Message} + return nil +} + +// update canaryStepPods, +func (r *RolloutHistoryReconciler) updateCanaryStepPods(RT *rolloutv1alpha1.Rollout, RH *rolloutv1alpha1.RolloutHistory) error { + podList := &corev1.PodList{} + var err error + var extraSelector labels.Selector + + // get labelSelector including rolloutBathID, rolloutID and workload selector + selector, _ := v1.LabelSelectorAsSelector(RH.Spec.Workload.Selector) + lableSelectorString := fmt.Sprintf("%v=%v,%v=%v,%v", util.RolloutBatchIDLabel, *RH.Status.CanaryStepIndex, util.RolloutIDLabel, RH.Spec.RolloutWrapper.Rollout.Spec.RolloutID, selector.String()) + extraSelector, err = labels.Parse(lableSelectorString) + if err != nil { + return err + } + + // get pods according to lableSelector + err = r.List(context.TODO(), podList, &client.ListOptions{LabelSelector: extraSelector}, client.InNamespace(RT.Namespace)) + if err != nil { + return err + } + + // if num of pods is empty, append a empty CanaryStepPods{} + if len(podList.Items) == 0 { + RH.Status.CanaryStepPods = append(RH.Status.CanaryStepPods, rolloutv1alpha1.CanaryStepPods{}) + return nil + } + + // get currentStepPods + currentStepPods := rolloutv1alpha1.CanaryStepPods{} + var pods []rolloutv1alpha1.Pod + // get pods name, ip, node and add them to CanaryStepPods.Pods + for i := range podList.Items { + pod := &podList.Items[i] + if pod.DeletionTimestamp.IsZero() { + cur := rolloutv1alpha1.Pod{Name: pod.Name, IP: pod.Status.PodIP, Node: pod.Spec.NodeName} + pods = append(pods, cur) + } + } + currentStepPods.Pods = pods + currentStepPods.StepIndex = *RH.Status.CanaryStepIndex + currentStepPods.PodsInStep = int32(len(pods)) + + // sum pods released by now + currentStepPods.PodsInTotal = int32(len(pods)) + for _, stepStatus := range RH.Status.CanaryStepPods { + currentStepPods.PodsInTotal += stepStatus.PodsInStep + } + + // add currentStepPods to .status.canaryStepPods + RH.Status.CanaryStepPods = append(RH.Status.CanaryStepPods, currentStepPods) + + return nil +} diff --git a/pkg/controller/rollouthistory/utils.go b/pkg/controller/rollouthistory/utils.go new file mode 100644 index 0000000..e4a1846 --- /dev/null +++ b/pkg/controller/rollouthistory/utils.go @@ -0,0 +1,39 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollouthistory + +import ( + "crypto/rand" + "math/big" + "strings" +) + +var CHARS = []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", + "1", "2", "3", "4", "5", "6", "7", "8", "9", "0"} + +func RandAllString(lenNum int) string { + str := strings.Builder{} + for i := 0; i < lenNum; i++ { + n, err := rand.Int(rand.Reader, big.NewInt(36)) + if err != nil { + return "" + } + l := CHARS[n.Int64()] + str.WriteString(l) + } + return str.String() +} diff --git a/test/e2e/rollouthistory_test.go b/test/e2e/rollouthistory_test.go new file mode 100644 index 0000000..c4a7ee5 --- /dev/null +++ b/test/e2e/rollouthistory_test.go @@ -0,0 +1,1894 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "errors" + "fmt" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + appsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" + appsv1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + utilpointer "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = SIGDescribe("RolloutHistory", func() { + var namespace string + + DumpAllResources := func() { + rollout := &rolloutsv1alpha1.RolloutList{} + k8sClient.List(context.TODO(), rollout, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(rollout)) + batch := &rolloutsv1alpha1.BatchReleaseList{} + k8sClient.List(context.TODO(), batch, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(batch)) + deploy := &apps.DeploymentList{} + k8sClient.List(context.TODO(), deploy, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(deploy)) + rs := &apps.ReplicaSetList{} + k8sClient.List(context.TODO(), rs, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(rs)) + cloneSet := &appsv1alpha1.CloneSetList{} + k8sClient.List(context.TODO(), cloneSet, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(cloneSet)) + sts := &apps.StatefulSetList{} + k8sClient.List(context.TODO(), sts, client.InNamespace(namespace)) + fmt.Println(util.DumpJSON(sts)) + } + + CreateObject := func(object client.Object, options ...client.CreateOption) { + object.SetNamespace(namespace) + Expect(k8sClient.Create(context.TODO(), object)).NotTo(HaveOccurred()) + } + + GetObject := func(name string, object client.Object) error { + key := types.NamespacedName{Namespace: namespace, Name: name} + return k8sClient.Get(context.TODO(), key, object) + } + /* + UpdateDeployment := func(object *apps.Deployment) *apps.Deployment { + var clone *apps.Deployment + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &apps.Deployment{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + clone.Spec.Template = *object.Spec.Template.DeepCopy() + clone.Labels = mergeMap(clone.Labels, object.Labels) + clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + */ + UpdateCloneSet := func(object *appsv1alpha1.CloneSet) *appsv1alpha1.CloneSet { + var clone *appsv1alpha1.CloneSet + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &appsv1alpha1.CloneSet{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + clone.Spec.Template = *object.Spec.Template.DeepCopy() + clone.Labels = mergeMap(clone.Labels, object.Labels) + clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + + UpdateRollout := func(object *rolloutsv1alpha1.Rollout) *rolloutsv1alpha1.Rollout { + var clone *rolloutsv1alpha1.Rollout + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &rolloutsv1alpha1.Rollout{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec = *object.Spec.DeepCopy() + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + + UpdateAdvancedStatefulSet := func(object *appsv1beta1.StatefulSet) *appsv1beta1.StatefulSet { + var clone *appsv1beta1.StatefulSet + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &appsv1beta1.StatefulSet{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + clone.Spec.Template = *object.Spec.Template.DeepCopy() + clone.Labels = mergeMap(clone.Labels, object.Labels) + clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + + UpdateNativeStatefulSet := func(object *apps.StatefulSet) *apps.StatefulSet { + var clone *apps.StatefulSet + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + clone = &apps.StatefulSet{} + err := GetObject(object.Name, clone) + if err != nil { + return err + } + clone.Spec.Replicas = utilpointer.Int32(*object.Spec.Replicas) + clone.Spec.Template = *object.Spec.Template.DeepCopy() + clone.Labels = mergeMap(clone.Labels, object.Labels) + clone.Annotations = mergeMap(clone.Annotations, object.Annotations) + return k8sClient.Update(context.TODO(), clone) + })).NotTo(HaveOccurred()) + + return clone + } + + ResumeRolloutCanary := func(name string) { + Eventually(func() bool { + clone := &rolloutsv1alpha1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStatus.CurrentStepState != rolloutsv1alpha1.CanaryStepStatePaused { + fmt.Println("resume rollout success, and CurrentStepState", util.DumpJSON(clone.Status)) + return true + } + + body := fmt.Sprintf(`{"status":{"canaryStatus":{"currentStepState":"%s"}}}`, rolloutsv1alpha1.CanaryStepStateReady) + Expect(k8sClient.Status().Patch(context.TODO(), clone, client.RawPatch(types.MergePatchType, []byte(body)))).NotTo(HaveOccurred()) + return false + }, 10*time.Second, time.Second).Should(BeTrue()) + } + /* + WaitDeploymentAllPodsReady := func(deployment *apps.Deployment) { + Eventually(func() bool { + clone := &apps.Deployment{} + Expect(GetObject(deployment.Name, clone)).NotTo(HaveOccurred()) + return clone.Status.ObservedGeneration == clone.Generation && *clone.Spec.Replicas == clone.Status.UpdatedReplicas && + *clone.Spec.Replicas == clone.Status.ReadyReplicas && *clone.Spec.Replicas == clone.Status.Replicas + }, 5*time.Minute, time.Second).Should(BeTrue()) + } + */ + WaitCloneSetAllPodsReady := func(cloneset *appsv1alpha1.CloneSet) { + Eventually(func() bool { + clone := &appsv1alpha1.CloneSet{} + Expect(GetObject(cloneset.Name, clone)).NotTo(HaveOccurred()) + return clone.Status.ObservedGeneration == clone.Generation && *clone.Spec.Replicas == clone.Status.UpdatedReplicas && + *clone.Spec.Replicas == clone.Status.ReadyReplicas && *clone.Spec.Replicas == clone.Status.Replicas + }, 5*time.Minute, time.Second).Should(BeTrue()) + } + + WaitNativeStatefulSetPodsReady := func(statefulset *apps.StatefulSet) { + Eventually(func() bool { + set := &apps.StatefulSet{} + Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) + return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && + *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas + }, 20*time.Minute, 3*time.Second).Should(BeTrue()) + } + + WaitAdvancedStatefulSetPodsReady := func(statefulset *appsv1beta1.StatefulSet) { + Eventually(func() bool { + set := &appsv1beta1.StatefulSet{} + Expect(GetObject(statefulset.Name, set)).NotTo(HaveOccurred()) + return set.Status.ObservedGeneration == set.Generation && *set.Spec.Replicas == set.Status.UpdatedReplicas && + *set.Spec.Replicas == set.Status.ReadyReplicas && *set.Spec.Replicas == set.Status.Replicas + }, 20*time.Minute, 3*time.Second).Should(BeTrue()) + } + /* + WaitDeploymentReplicas := func(deployment *apps.Deployment) { + Eventually(func() bool { + clone := &apps.Deployment{} + Expect(GetObject(deployment.Name, clone)).NotTo(HaveOccurred()) + return clone.Status.ObservedGeneration == clone.Generation && + *clone.Spec.Replicas == clone.Status.ReadyReplicas && *clone.Spec.Replicas == clone.Status.Replicas + }, 10*time.Minute, time.Second).Should(BeTrue()) + } + */ + /* + WaitRolloutCanaryStepPaused := func(name string, stepIndex int32) { + start := time.Now() + Eventually(func() bool { + if start.Add(time.Minute * 5).Before(time.Now()) { + DumpAllResources() + Expect(true).Should(BeFalse()) + } + clone := &rolloutsv1alpha1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStatus == nil { + return false + } + klog.Infof("current step:%v target step:%v current step state %v", clone.Status.CanaryStatus.CurrentStepIndex, stepIndex, clone.Status.CanaryStatus.CurrentStepState) + return clone.Status.CanaryStatus.CurrentStepIndex == stepIndex && clone.Status.CanaryStatus.CurrentStepState == rolloutsv1alpha1.CanaryStepStatePaused + }, 20*time.Minute, time.Second).Should(BeTrue()) + } + */ + WaitRolloutStatusPhase := func(name string, phase rolloutsv1alpha1.RolloutPhase) { + Eventually(func() bool { + clone := &rolloutsv1alpha1.Rollout{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + return clone.Status.Phase == phase + }, 20*time.Minute, time.Second).Should(BeTrue()) + } + + ListPods := func(namespace string, selector labels.Selector) ([]*v1.Pod, error) { + appList := &v1.PodList{} + err := k8sClient.List(context.TODO(), appList, &client.ListOptions{Namespace: namespace, LabelSelector: selector}) + if err != nil { + return nil, err + } + apps := make([]*v1.Pod, 0) + for i := range appList.Items { + pod := &appList.Items[i] + if pod.DeletionTimestamp.IsZero() { + apps = append(apps, pod) + } + } + return apps, nil + } + + CheckPodsBatchLabel := func(namespace string, labelSelector *metav1.LabelSelector, rolloutID, batchID string, expected int) { + selector, _ := metav1.LabelSelectorAsSelector(labelSelector) + + lableSelectorString := fmt.Sprintf("%v=%v,%v=%v,%v", util.RolloutBatchIDLabel, batchID, util.RolloutIDLabel, rolloutID, selector.String()) + extraSelector, err := labels.Parse(lableSelectorString) + Expect(err).NotTo(HaveOccurred()) + + pods, err := ListPods(namespace, extraSelector) + Expect(err).NotTo(HaveOccurred()) + + Expect(len(pods)).Should(BeNumerically("==", expected)) + } + + CheckRolloutHistoryPodsBatchLabel := func(pods *rolloutsv1alpha1.CanaryStepPods, rolloutID, batchID string) bool { + for _, podInfo := range pods.Pods { + pod := &v1.Pod{} + Expect(GetObject(podInfo.Name, pod)) + if pod.Labels[util.RolloutIDLabel] != rolloutID && + pod.Labels[util.RolloutBatchIDLabel] != batchID { + return false + } + } + + return true + } + + GetRolloutHistory := func(rollout *rolloutsv1alpha1.Rollout) (RH *rolloutsv1alpha1.RolloutHistory, err error) { + Eventually(func() bool { + rhList := &rolloutsv1alpha1.RolloutHistoryList{} + Expect(k8sClient.List(context.TODO(), rhList, &client.ListOptions{}, client.InNamespace(rollout.Namespace))).NotTo(HaveOccurred()) + + result := rhList.Items + + for i := range result { + RH = &result[i] + if rollout.Spec.RolloutID == RH.Spec.RolloutWrapper.Rollout.Spec.RolloutID && + rollout.Name == RH.Spec.RolloutWrapper.Name { + err = nil + return true + } + } + + err = errors.New("rollout history not found") + return false + }, 30*time.Second, time.Second).Should(BeTrue()) + return + } + + ListRolloutHistories := func(namespace string) ([]*rolloutsv1alpha1.RolloutHistory, error) { + rhList := &rolloutsv1alpha1.RolloutHistoryList{} + err := k8sClient.List(context.TODO(), rhList, &client.ListOptions{Namespace: namespace}) + if err != nil { + return nil, err + } + rhs := make([]*rolloutsv1alpha1.RolloutHistory, 0) + for i := range rhList.Items { + rh := &rhList.Items[i] + if rh.DeletionTimestamp.IsZero() { + rhs = append(rhs, rh) + } + } + return rhs, nil + } + + WaitRolloutHistoryPhase := func(name string, status string) { + start := time.Now() + Eventually(func() bool { + if start.Add(time.Minute * 5).Before(time.Now()) { + DumpAllResources() + Expect(true).Should(BeFalse()) + } + clone := &rolloutsv1alpha1.RolloutHistory{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStepIndex == nil { + return false + } + klog.Infof("rolloutID:%v current rhphase:%v current step: %v current status:%v target step:%v ", clone.Spec.RolloutWrapper.Rollout.Spec.RolloutID, clone.Status.CanaryStepState, clone.Status.CanaryStepIndex, clone.Status.Phase, status) + return clone.Status.Phase == status + }, 20*time.Minute, time.Second).Should(BeTrue()) + } + + WaitRolloutHistoryStepPaused := func(name string, stepIndex int32) { + start := time.Now() + Eventually(func() bool { + if start.Add(time.Minute * 5).Before(time.Now()) { + DumpAllResources() + Expect(true).Should(BeFalse()) + } + clone := &rolloutsv1alpha1.RolloutHistory{} + Expect(GetObject(name, clone)).NotTo(HaveOccurred()) + if clone.Status.CanaryStepIndex == nil { + return false + } + klog.Infof("current step:%v target step:%v current step state %v", *clone.Status.CanaryStepIndex, stepIndex, clone.Status.CanaryStepState) + return *clone.Status.CanaryStepIndex == stepIndex && clone.Status.CanaryStepState == rolloutsv1alpha1.CanaryStateUpdated + }, 20*time.Minute, time.Second).Should(BeTrue()) + } + + BeforeEach(func() { + namespace = randomNamespaceName("rollout") + ns := v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + Expect(k8sClient.Create(context.TODO(), &ns)).Should(SatisfyAny(BeNil())) + }) + + AfterEach(func() { + By("[TEST] Clean up resources after an integration test") + k8sClient.DeleteAllOf(context.TODO(), &apps.Deployment{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &appsv1alpha1.CloneSet{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &rolloutsv1alpha1.BatchRelease{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &rolloutsv1alpha1.Rollout{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &v1.Service{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &netv1.Ingress{}, client.InNamespace(namespace)) + k8sClient.DeleteAllOf(context.TODO(), &rolloutsv1alpha1.RolloutHistory{}, client.InNamespace(namespace)) + Expect(k8sClient.Delete(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, client.PropagationPolicy(metav1.DeletePropagationForeground))).Should(Succeed()) + time.Sleep(time.Second * 3) + }) + + KruiseDescribe("CloneSet canary rollout with RolloutHistory", func() { + It("V1->V2: Percentage, 20%,60% Succeeded", func() { + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(20), + Pause: rolloutsv1alpha1.RolloutPause{Duration: utilpointer.Int32(10)}, + }, + { + Weight: utilpointer.Int32(60), + Pause: rolloutsv1alpha1.RolloutPause{Duration: utilpointer.Int32(10)}, + }, + } + rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + rollout.Spec.RolloutID = "1" + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "2" + UpdateRollout(rollout) + UpdateCloneSet(workload) + By("Update rollouthistory rolloutID from(1) -> to(2), update cloneSet env NODE_NAME from(version1) -> to(version2)") + + // wait step 1 complete + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err := GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + + // check out the num of rollouthistory + rhs, err := ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 1)) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status & paused + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 1)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + By("check rollouthistory status & update success") + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutHistoryStepPaused(rollouthistory.Name, 2) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + + // check rollouthistory status & paused + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 2)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 2)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 2) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 2)) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + By("check rollouthistory status & update success") + + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitCloneSetAllPodsReady(workload) + By("rollout completed, and check") + + WaitRolloutHistoryPhase(rollouthistory.Name, rolloutsv1alpha1.PhaseCompleted) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateCompleted)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + + // check rollouthistory status & paused + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 3)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 3)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateCompleted)) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 2)) + Expect(len(rollouthistory.Status.CanaryStepPods[2].Pods)).Should(BeNumerically("==", 0)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 2) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + /// the last rollout + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "3", 0) + /// Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.StepStatus[2].PodList, rollout.Spec.RolloutID, "3")).Should(BeTrue()) + fmt.Println(rollouthistory.Status.CanaryStepPods) /// dialog + fmt.Println() + fmt.Println() + + // check cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + selector, _ := metav1.LabelSelectorAsSelector(workload.Spec.Selector) + fmt.Println(ListPods(namespace, selector)) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + By("check rollouthistory status & update success") + }) + + It("V1->V2: Percentage, 20%, and rollback(v1)", func() { + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + stableRevision := rollout.Status.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Image = "echoserver:failed" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "1" + UpdateRollout(rollout) + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2), rolloutID from('') -> to(1)") + // wait step 1 complete + time.Sleep(time.Second * 20) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 0)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateUpgrade)) + + // check rollouthistory + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err := GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + + // check rollouthistory num + rhs, err := ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 1)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 0)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStatePending)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + + // rollback -> v1 + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:1.10.2" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "2" + UpdateRollout(rollout) + UpdateCloneSet(workload) + By("Rollback deployment env NODE_NAME from(version2) -> to(version1), rolloutID from(1) -> to(2)") + time.Sleep(time.Second * 5) + + // check rollouthistory + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err = GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + ResumeRolloutCanary(rollout.Name) + + WaitRolloutHistoryPhase(rollouthistory.Name, rolloutsv1alpha1.PhaseCompleted) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateCompleted)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 5)) + + // check rollouthistory num + rhs, err = ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 2)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 5)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 5)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateCompleted)) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[2].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[3].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[4].Pods)).Should(BeNumerically("==", 1)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "3", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "4", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "5", 1) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[2], rollout.Spec.RolloutID, "3")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[3], rollout.Spec.RolloutID, "4")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[4], rollout.Spec.RolloutID, "5")).Should(BeTrue()) + + By("rollout completed, and check") + // check workload + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Spec.UpdateStrategy.Partition.IntVal).Should(BeNumerically("==", 0)) + + }) + + It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1alpha1.CloneSet{} + Expect(ReadYamlToObject("./test_data/rollout/cloneset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitCloneSetAllPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "1" + UpdateRollout(rollout) + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err := GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Spec.UpdateStrategy.Paused).Should(BeFalse()) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + + // check rollouthistory num + rhs, err := ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 1)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 1)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + By("check rollouthistory[rollout-id:1 batch-id:1] status & update success") + + // resume rollouthistory canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + + // v1 -> v2 -> v3, continuous release + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "2" + UpdateRollout(rollout) + UpdateCloneSet(workload) + By("Update cloneSet env NODE_NAME from(version2) -> to(version3), rollout from(1) -> to(2)") + time.Sleep(time.Second * 10) + + // wait step 1 complete + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err = GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + + // check rollouthistory num + rhs, err = ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 2)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 1)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + By("check rollouthistory[rollout-id:2 batch-id:1] status & update success") + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("check rollout canary status success, resume rollout, and wait rollout canary complete") + WaitRolloutHistoryPhase(rollouthistory.Name, rolloutsv1alpha1.PhaseCompleted) + By("rollout completed, and check") + + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.UpdatedReadyReplicas).Should(BeNumerically("==", 5)) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 5)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 5)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateCompleted)) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[2].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[3].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[4].Pods)).Should(BeNumerically("==", 1)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "3", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "4", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "5", 1) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[2], rollout.Spec.RolloutID, "3")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[3], rollout.Spec.RolloutID, "4")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[4], rollout.Spec.RolloutID, "5")).Should(BeTrue()) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + }) + }) + + KruiseDescribe("StatefulSet canary rollout with RolloutHistory", func() { + KruiseDescribe("Native StatefulSet canary rollout with RolloutHistory", func() { + It("V1->V2: Percentage, 20%,60% Succeeded", func() { + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(20), + Pause: rolloutsv1alpha1.RolloutPause{}, + }, + { + Weight: utilpointer.Int32(60), + Pause: rolloutsv1alpha1.RolloutPause{}, + }, + } + rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "echoserver", + } + rollout.Spec.RolloutID = "1" + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // headless service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitNativeStatefulSetPodsReady(workload) + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "2" + UpdateRollout(rollout) + UpdateNativeStatefulSet(workload) + By("Update rollouthistory rolloutID from(1) -> to(2), Update NativeStatefulSet env NODE_NAME from(version1) -> to(version2)") + + // wait step 1 complete + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err := GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + + // check out the num of rollouthistory + rhs, err := ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 1)) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status & paused + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 1)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + By("check rollouthistory status & update success") + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + By("check statefulset status & paused success") + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutHistoryStepPaused(rollouthistory.Name, 2) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + + // check rollouthistory status & paused + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 2)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 2)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 2) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 2)) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + By("check rollouthistory status & update success") + + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + By("check cloneSet status & paused success") + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitNativeStatefulSetPodsReady(workload) + By("rollout completed, and check") + + WaitRolloutHistoryPhase(rollouthistory.Name, rolloutsv1alpha1.PhaseCompleted) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateCompleted)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + + // check rollouthistory status & paused + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 3)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 3)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateCompleted)) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 2)) + Expect(len(rollouthistory.Status.CanaryStepPods[2].Pods)).Should(BeNumerically("==", 0)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 2) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + /// the last rollout + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "3", 0) + /// Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.StepStatus[2].PodList, rollout.Spec.RolloutID, "3")).Should(BeTrue()) + fmt.Println(rollouthistory.Status.CanaryStepPods) /// dialog + fmt.Println() + fmt.Println() + + // check cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + selector, _ := metav1.LabelSelectorAsSelector(workload.Spec.Selector) + fmt.Println(ListPods(namespace, selector)) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas)) + By("check cloneSet status & paused success") + By("check rollouthistory status & update success") + }) + + It("V1->V2: Percentage, 20%, and rollback(v1)", func() { + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // headless service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitNativeStatefulSetPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + stableRevision := rollout.Status.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Image = "echoserver:failed" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "1" + UpdateRollout(rollout) + UpdateNativeStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2), rolloutID from('') -> to(1)") + + // wait step 1 complete + time.Sleep(time.Second * 20) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateUpgrade)) + + // check rollouthistory + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err := GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + + // check rollouthistory num + rhs, err := ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 1)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 0)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStatePending)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + + // rollback -> v1 + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:1.10.2" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "2" + UpdateRollout(rollout) + UpdateNativeStatefulSet(workload) + By("Rollback deployment env NODE_NAME from(version2) -> to(version1), rolloutID from(1) -> to(2)") + time.Sleep(time.Second * 5) + + // StatefulSet will not remove the broken pod with failed image, we should delete it manually + brokenPod := &v1.Pod{} + Expect(GetObject(fmt.Sprintf("%v-%v", workload.Name, *workload.Spec.Replicas-1), brokenPod)).NotTo(HaveOccurred()) + Expect(k8sClient.Delete(context.TODO(), brokenPod)).NotTo(HaveOccurred()) + + // check rollouthistory + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err = GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + ResumeRolloutCanary(rollout.Name) + + WaitRolloutHistoryPhase(rollouthistory.Name, rolloutsv1alpha1.PhaseCompleted) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateCompleted)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 5)) + + // check rollouthistory num + rhs, err = ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 2)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 5)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 5)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateCompleted)) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[2].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[3].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[4].Pods)).Should(BeNumerically("==", 1)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "3", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "4", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "5", 1) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[2], rollout.Spec.RolloutID, "3")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[3], rollout.Spec.RolloutID, "4")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[4], rollout.Spec.RolloutID, "5")).Should(BeTrue()) + + By("rollout completed, and check") + // check workload + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + Expect(*workload.Spec.UpdateStrategy.RollingUpdate.Partition).Should(BeNumerically("==", 0)) + + }) + + It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // headless service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &apps.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/native_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitNativeStatefulSetPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "1" + UpdateRollout(rollout) + UpdateNativeStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err := GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + + // check rollouthistory num + rhs, err := ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 1)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 1)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + By("check rollouthistory[rollout-id:1 batch-id:1] status & update success") + + // resume rollouthistory canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + + // v1 -> v2 -> v3, continuous release + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "2" + UpdateRollout(rollout) + UpdateNativeStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version2) -> to(version3), rollout from(1) -> to(2)") + time.Sleep(time.Second * 10) + + // wait step 1 complete + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err = GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + + // check rollouthistory num + rhs, err = ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 2)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 1)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + By("check rollouthistory[rollout-id:2 batch-id:1] status & update success") + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("check rollout canary status success, resume rollout, and wait rollout canary complete") + WaitRolloutHistoryPhase(rollouthistory.Name, rolloutsv1alpha1.PhaseCompleted) + By("rollout completed, and check") + + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 5)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 5)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateCompleted)) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[2].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[3].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[4].Pods)).Should(BeNumerically("==", 1)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "3", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "4", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "5", 1) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[2], rollout.Spec.RolloutID, "3")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[3], rollout.Spec.RolloutID, "4")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[4], rollout.Spec.RolloutID, "5")).Should(BeTrue()) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + }) + }) + + KruiseDescribe("Advanced StatefulSet canary rollout with RolloutHistory", func() { + It("V1->V2: Percentage, 20%,60% Succeeded", func() { + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(20), + Pause: rolloutsv1alpha1.RolloutPause{}, + }, + { + Weight: utilpointer.Int32(60), + Pause: rolloutsv1alpha1.RolloutPause{}, + }, + } + rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + APIVersion: "apps.kruise.io/v1beta1", + Kind: "StatefulSet", + Name: "echoserver", + } + rollout.Spec.RolloutID = "1" + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // headless service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1beta1.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitAdvancedStatefulSetPodsReady(workload) + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "2" + UpdateRollout(rollout) + UpdateAdvancedStatefulSet(workload) + By("Update rollouthistory rolloutID from(1) -> to(2), update cloneSet env NODE_NAME from(version1) -> to(version2)") + + // wait step 1 complete + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err := GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + + // check out the num of rollouthistory + rhs, err := ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 1)) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status & paused + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 1)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + By("check rollouthistory status & update success") + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + By("check cloneSet status & paused success") + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("resume rollout, and wait next step(2)") + WaitRolloutHistoryStepPaused(rollouthistory.Name, 2) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + + // check rollouthistory status & paused + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 2)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 2)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 2) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 2)) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + By("check rollouthistory status & update success") + + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 3)) + By("check cloneSet status & paused success") + + // resume rollout + ResumeRolloutCanary(rollout.Name) + WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy) + WaitAdvancedStatefulSetPodsReady(workload) + By("rollout completed, and check") + + WaitRolloutHistoryPhase(rollouthistory.Name, rolloutsv1alpha1.PhaseCompleted) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateCompleted)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 2)) + + // check rollouthistory status & paused + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 3)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 3)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateCompleted)) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 2)) + Expect(len(rollouthistory.Status.CanaryStepPods[2].Pods)).Should(BeNumerically("==", 0)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 2) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + /// the last rollout + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "3", 0) + /// Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.StepStatus[2].PodList, rollout.Spec.RolloutID, "3")).Should(BeTrue()) + fmt.Println(rollouthistory.Status.CanaryStepPods) /// dialog + fmt.Println() + fmt.Println() + + // check cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + selector, _ := metav1.LabelSelectorAsSelector(workload.Spec.Selector) + fmt.Println(ListPods(namespace, selector)) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + By("check cloneSet status & paused success") + By("check rollouthistory status & update success") + }) + + It("V1->V2: Percentage, 20%, and rollback(v1)", func() { + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + APIVersion: "apps.kruise.io/v1beta1", + Kind: "StatefulSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + // service + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1beta1.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitAdvancedStatefulSetPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + stableRevision := rollout.Status.StableRevision + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Image = "echoserver:failed" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "1" + UpdateRollout(rollout) + UpdateAdvancedStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2), rolloutID from('') -> to(1)") + + // wait step 1 complete + time.Sleep(time.Second * 20) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 0)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.StableRevision).Should(Equal(stableRevision)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateUpgrade)) + + // check rollouthistory + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err := GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + + // check rollouthistory num + rhs, err := ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 1)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 0)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStatePending)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + + // rollback -> v1 + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version1"}) + workload.Spec.Template.Spec.Containers[0].Image = "cilium/echoserver:1.10.2" + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "2" + UpdateRollout(rollout) + UpdateAdvancedStatefulSet(workload) + By("Rollback deployment env NODE_NAME from(version2) -> to(version1), rolloutID from(1) -> to(2)") + time.Sleep(time.Second * 5) + + // StatefulSet will not remove the broken pod with failed image, we should delete it manually + brokenPod := &v1.Pod{} + Expect(GetObject(fmt.Sprintf("%v-%v", workload.Name, *workload.Spec.Replicas-1), brokenPod)).NotTo(HaveOccurred()) + Expect(k8sClient.Delete(context.TODO(), brokenPod)).NotTo(HaveOccurred()) + + // check rollouthistory + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err = GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + ResumeRolloutCanary(rollout.Name) + + WaitRolloutHistoryPhase(rollouthistory.Name, rolloutsv1alpha1.PhaseCompleted) + + // check rollout status & paused + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStateCompleted)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 5)) + + // check rollouthistory num + rhs, err = ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 2)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 5)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 5)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateCompleted)) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[2].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[3].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[4].Pods)).Should(BeNumerically("==", 1)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "3", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "4", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "5", 1) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[2], rollout.Spec.RolloutID, "3")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[3], rollout.Spec.RolloutID, "4")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[4], rollout.Spec.RolloutID, "5")).Should(BeTrue()) + + By("rollout completed, and check") + // check workload + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + + }) + + It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { + By("Creating Rollout...") + rollout := &rolloutsv1alpha1.Rollout{} + Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) + rollout.Spec.ObjectRef.WorkloadRef = &rolloutsv1alpha1.WorkloadRef{ + APIVersion: "apps.kruise.io/v1beta1", + Kind: "StatefulSet", + Name: "echoserver", + } + CreateObject(rollout) + + By("Creating workload and waiting for all pods ready...") + headlessService := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/headless_service.yaml", headlessService)).ToNot(HaveOccurred()) + CreateObject(headlessService) + // service + service := &v1.Service{} + Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred()) + CreateObject(service) + // ingress + ingress := &netv1.Ingress{} + Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred()) + CreateObject(ingress) + // workload + workload := &appsv1beta1.StatefulSet{} + Expect(ReadYamlToObject("./test_data/rollout/advanced_statefulset.yaml", workload)).ToNot(HaveOccurred()) + CreateObject(workload) + WaitAdvancedStatefulSetPodsReady(workload) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseHealthy)) + By("check rollout status & paused success") + + // v1 -> v2, start rollout action + newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "1" + UpdateRollout(rollout) + UpdateAdvancedStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version1) -> to(version2)") + // wait step 1 complete + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err := GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + + // check workload status & paused + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 1)) + By("check cloneSet status & paused success") + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + + // check rollouthistory num + rhs, err := ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 1)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 1)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + By("check rollouthistory[rollout-id:1 batch-id:1] status & update success") + + // resume rollouthistory canary + ResumeRolloutCanary(rollout.Name) + time.Sleep(time.Second * 15) + + // v1 -> v2 -> v3, continuous release + newEnvs = mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version3"}) + workload.Spec.Template.Spec.Containers[0].Env = newEnvs + rollout.Spec.RolloutID = "2" + UpdateRollout(rollout) + UpdateAdvancedStatefulSet(workload) + By("Update cloneSet env NODE_NAME from(version2) -> to(version3), rollout from(1) -> to(2)") + time.Sleep(time.Second * 10) + + // wait step 1 complete + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + rollouthistory, err = GetRolloutHistory(rollout) + Expect(err).To(BeNil()) + WaitRolloutHistoryStepPaused(rollouthistory.Name, 1) + + // check rollout status + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing)) + Expect(rollout.Status.CanaryStatus.CurrentStepState).Should(Equal(rolloutsv1alpha1.CanaryStepStatePaused)) + Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1)) + + // check rollouthistory num + rhs, err = ListRolloutHistories(namespace) + Expect(err).To(BeNil()) + Expect(len(rhs)).Should(BeNumerically("==", 2)) + + // check rollouthistory spec + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Name).Should(Equal(ingress.Name)) + Expect(rollouthistory.Spec.TrafficRoutingWrapper.IngressWrapper.Ingress).NotTo(BeNil()) + Expect(rollouthistory.Spec.ServiceWrapper.Name).Should(Equal(service.Name)) + Expect(rollouthistory.Spec.ServiceWrapper.Service).NotTo(BeNil()) + Expect(rollouthistory.Spec.Workload.Name).Should(Equal(workload.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Name).Should(Equal(rollout.Name)) + Expect(rollouthistory.Spec.RolloutWrapper.Rollout).NotTo(BeNil()) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 1)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateUpdated)) + Expect(rollouthistory.Status.Phase).Should(Equal(rolloutsv1alpha1.PhaseProgressing)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + By("check rollouthistory[rollout-id:2 batch-id:1] status & update success") + + // resume rollout canary + ResumeRolloutCanary(rollout.Name) + By("check rollout canary status success, resume rollout, and wait rollout canary complete") + WaitRolloutHistoryPhase(rollouthistory.Name, rolloutsv1alpha1.PhaseCompleted) + By("rollout completed, and check") + + // cloneset + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", 5)) + + // check rollouthistory status + Expect(GetObject(rollouthistory.Name, rollouthistory)).NotTo(HaveOccurred()) + Expect(*rollouthistory.Status.CanaryStepIndex).Should(BeNumerically("==", 5)) + Expect(len(rollouthistory.Status.CanaryStepPods)).Should(BeNumerically("==", 5)) + Expect(rollouthistory.Status.CanaryStepState).Should(Equal(rolloutsv1alpha1.CanaryStateCompleted)) + Expect(len(rollouthistory.Status.CanaryStepPods[0].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[1].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[2].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[3].Pods)).Should(BeNumerically("==", 1)) + Expect(len(rollouthistory.Status.CanaryStepPods[4].Pods)).Should(BeNumerically("==", 1)) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "1", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "2", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "3", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "4", 1) + CheckPodsBatchLabel(namespace, workload.Spec.Selector, rollout.Spec.RolloutID, "5", 1) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[0], rollout.Spec.RolloutID, "1")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[1], rollout.Spec.RolloutID, "2")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[2], rollout.Spec.RolloutID, "3")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[3], rollout.Spec.RolloutID, "4")).Should(BeTrue()) + Expect(CheckRolloutHistoryPodsBatchLabel(&rollouthistory.Status.CanaryStepPods[4], rollout.Spec.RolloutID, "5")).Should(BeTrue()) + + // check progressing succeed + Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) + Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred()) + cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing) + Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded)) + Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue))) + }) + }) + }) + +})