From b7315e165846ae830c8a93cd5fae06a8edbaf818 Mon Sep 17 00:00:00 2001 From: yike21 Date: Mon, 31 Oct 2022 11:50:23 +0800 Subject: [PATCH] [Issue10] Add_rollout history api and controller (#61) * add RolloutHistory api Signed-off-by: yike21 * add RolloutHistory controller Signed-off-by: yike21 Signed-off-by: yike21 --- api/v1alpha1/rollouthistory_types.go | 151 +++ api/v1alpha1/zz_generated.deepcopy.go | 259 +++- .../rollouts.kruise.io_rollouthistories.yaml | 176 +++ config/crd/kustomization.yaml | 1 + config/rbac/role.yaml | 26 + go.mod | 2 + go.sum | 8 - main.go | 12 +- .../rollouthistory_controller.go | 395 ++++++ .../rollouthistory_controller_test.go | 1062 +++++++++++++++++ .../rollouthistory_event_handler.go | 88 ++ pkg/controller/rollouthistory/util.go | 342 ++++++ pkg/feature/rollout_features.go | 37 + pkg/util/feature/feature_gate.go | 32 + 14 files changed, 2581 insertions(+), 10 deletions(-) create mode 100644 api/v1alpha1/rollouthistory_types.go create mode 100644 config/crd/bases/rollouts.kruise.io_rollouthistories.yaml create mode 100644 pkg/controller/rollouthistory/rollouthistory_controller.go create mode 100644 pkg/controller/rollouthistory/rollouthistory_controller_test.go create mode 100644 pkg/controller/rollouthistory/rollouthistory_event_handler.go create mode 100644 pkg/controller/rollouthistory/util.go create mode 100644 pkg/feature/rollout_features.go create mode 100644 pkg/util/feature/feature_gate.go diff --git a/api/v1alpha1/rollouthistory_types.go b/api/v1alpha1/rollouthistory_types.go new file mode 100644 index 0000000..0a80be3 --- /dev/null +++ b/api/v1alpha1/rollouthistory_types.go @@ -0,0 +1,151 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// RolloutHistorySpec defines the desired state of RolloutHistory +type RolloutHistorySpec struct { + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Rollout indicates information of the rollout related with rollouthistory + Rollout RolloutInfo `json:"rollout,omitempty"` + // Workload indicates information of the workload, such as cloneset, deployment, advanced statefulset + Workload WorkloadInfo `json:"workload,omitempty"` + // Service indicates information of the service related with workload + Service ServiceInfo `json:"service,omitempty"` + // TrafficRouting indicates information of traffic route related with workload + TrafficRouting TrafficRoutingInfo `json:"trafficRouting,omitempty"` +} + +type NameAndSpecData struct { + // Name indicates the name of object ref, such as rollout name, workload name, ingress name, etc. + Name string `json:"name"` + // Data indecates the spec of object ref + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Data runtime.RawExtension `json:"data,omitempty"` +} + +// RolloutInfo indicates information of the rollout related +type RolloutInfo struct { + // RolloutID indicates the new rollout + // if there is no new RolloutID this time, ignore it and not execute RolloutHistory + RolloutID string `json:"rolloutID"` + NameAndSpecData `json:",inline"` +} + +// ServiceInfo indicates information of the service related +type ServiceInfo struct { + NameAndSpecData `json:",inline"` +} + +// TrafficRoutingInfo indicates information of Gateway API or Ingress +type TrafficRoutingInfo struct { + // IngressRef indicates information of ingress + // +optional + Ingress *IngressInfo `json:"ingress,omitempty"` + // HTTPRouteRef indacates information of Gateway API + // +optional + HTTPRoute *HTTPRouteInfo `json:"httpRoute,omitempty"` +} + +// IngressInfo indicates information of the ingress related +type IngressInfo struct { + NameAndSpecData `json:",inline"` +} + +// HTTPRouteInfo indicates information of gateway API +type HTTPRouteInfo struct { + NameAndSpecData `json:",inline"` +} + +// WorkloadInfo indicates information of the workload, such as cloneset, deployment, advanced statefulset +type WorkloadInfo struct { + metav1.TypeMeta `json:",inline"` + NameAndSpecData `json:",inline"` +} + +// RolloutHistoryStatus defines the observed state of RolloutHistory +type RolloutHistoryStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // Phase indicates phase of RolloutHistory, just "" or "completed" + Phase string `json:"phase,omitempty"` + // CanarySteps indicates the pods released each step + CanarySteps []CanaryStepInfo `json:"canarySteps,omitempty"` +} + +// CanaryStepInfo indicates the pods for a revision +type CanaryStepInfo struct { + // CanaryStepIndex indicates step this revision + CanaryStepIndex int32 `json:"canaryStepIndex,omitempty"` + // Pods indicates the pods information + Pods []Pod `json:"pods,omitempty"` +} + +// Pod indicates the information of a pod, including name, ip, node_name. +type Pod struct { + // Name indicates the node name + Name string `json:"name,omitempty"` + // IP indicates the pod ip + IP string `json:"ip,omitempty"` + // NodeName indicates the node which pod is located at + NodeName string `json:"nodeName,omitempty"` + // todo + // State indicates whether the pod is ready or not + // State string `json:"state, omitempty"` +} + +// Phase indicates rollouthistory phase +const ( + PhaseCompleted string = "completed" +) + +// +genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// RolloutHistory is the Schema for the rollouthistories API +type RolloutHistory struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec RolloutHistorySpec `json:"spec,omitempty"` + Status RolloutHistoryStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// RolloutHistoryList contains a list of RolloutHistory +type RolloutHistoryList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []RolloutHistory `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RolloutHistory{}, &RolloutHistoryList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 39eb6a6..ff60b06 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ limitations under the License. package v1alpha1 import ( - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -199,6 +199,26 @@ func (in *CanaryStep) DeepCopy() *CanaryStep { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CanaryStepInfo) DeepCopyInto(out *CanaryStepInfo) { + *out = *in + if in.Pods != nil { + in, out := &in.Pods, &out.Pods + *out = make([]Pod, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryStepInfo. +func (in *CanaryStepInfo) DeepCopy() *CanaryStepInfo { + if in == nil { + return nil + } + out := new(CanaryStepInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CanaryStrategy) DeepCopyInto(out *CanaryStrategy) { *out = *in @@ -252,6 +272,38 @@ func (in *GatewayTrafficRouting) DeepCopy() *GatewayTrafficRouting { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPRouteInfo) DeepCopyInto(out *HTTPRouteInfo) { + *out = *in + in.NameAndSpecData.DeepCopyInto(&out.NameAndSpecData) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPRouteInfo. +func (in *HTTPRouteInfo) DeepCopy() *HTTPRouteInfo { + if in == nil { + return nil + } + out := new(HTTPRouteInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IngressInfo) DeepCopyInto(out *IngressInfo) { + *out = *in + in.NameAndSpecData.DeepCopyInto(&out.NameAndSpecData) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressInfo. +func (in *IngressInfo) DeepCopy() *IngressInfo { + if in == nil { + return nil + } + out := new(IngressInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressTrafficRouting) DeepCopyInto(out *IngressTrafficRouting) { *out = *in @@ -267,6 +319,22 @@ func (in *IngressTrafficRouting) DeepCopy() *IngressTrafficRouting { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NameAndSpecData) DeepCopyInto(out *NameAndSpecData) { + *out = *in + in.Data.DeepCopyInto(&out.Data) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NameAndSpecData. +func (in *NameAndSpecData) DeepCopy() *NameAndSpecData { + if in == nil { + return nil + } + out := new(NameAndSpecData) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectRef) DeepCopyInto(out *ObjectRef) { *out = *in @@ -287,6 +355,21 @@ func (in *ObjectRef) DeepCopy() *ObjectRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Pod) DeepCopyInto(out *Pod) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pod. +func (in *Pod) DeepCopy() *Pod { + if in == nil { + return nil + } + out := new(Pod) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReleaseBatch) DeepCopyInto(out *ReleaseBatch) { *out = *in @@ -372,6 +455,122 @@ func (in *RolloutCondition) DeepCopy() *RolloutCondition { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutHistory) DeepCopyInto(out *RolloutHistory) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutHistory. +func (in *RolloutHistory) DeepCopy() *RolloutHistory { + if in == nil { + return nil + } + out := new(RolloutHistory) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RolloutHistory) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutHistoryList) DeepCopyInto(out *RolloutHistoryList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RolloutHistory, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutHistoryList. +func (in *RolloutHistoryList) DeepCopy() *RolloutHistoryList { + if in == nil { + return nil + } + out := new(RolloutHistoryList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RolloutHistoryList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutHistorySpec) DeepCopyInto(out *RolloutHistorySpec) { + *out = *in + in.Rollout.DeepCopyInto(&out.Rollout) + in.Workload.DeepCopyInto(&out.Workload) + in.Service.DeepCopyInto(&out.Service) + in.TrafficRouting.DeepCopyInto(&out.TrafficRouting) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutHistorySpec. +func (in *RolloutHistorySpec) DeepCopy() *RolloutHistorySpec { + if in == nil { + return nil + } + out := new(RolloutHistorySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutHistoryStatus) DeepCopyInto(out *RolloutHistoryStatus) { + *out = *in + if in.CanarySteps != nil { + in, out := &in.CanarySteps, &out.CanarySteps + *out = make([]CanaryStepInfo, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutHistoryStatus. +func (in *RolloutHistoryStatus) DeepCopy() *RolloutHistoryStatus { + if in == nil { + return nil + } + out := new(RolloutHistoryStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutInfo) DeepCopyInto(out *RolloutInfo) { + *out = *in + in.NameAndSpecData.DeepCopyInto(&out.NameAndSpecData) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutInfo. +func (in *RolloutInfo) DeepCopy() *RolloutInfo { + if in == nil { + return nil + } + out := new(RolloutInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RolloutList) DeepCopyInto(out *RolloutList) { *out = *in @@ -488,6 +687,22 @@ func (in *RolloutStrategy) DeepCopy() *RolloutStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceInfo) DeepCopyInto(out *ServiceInfo) { + *out = *in + in.NameAndSpecData.DeepCopyInto(&out.NameAndSpecData) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceInfo. +func (in *ServiceInfo) DeepCopy() *ServiceInfo { + if in == nil { + return nil + } + out := new(ServiceInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TrafficRouting) DeepCopyInto(out *TrafficRouting) { *out = *in @@ -513,6 +728,48 @@ func (in *TrafficRouting) DeepCopy() *TrafficRouting { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TrafficRoutingInfo) DeepCopyInto(out *TrafficRoutingInfo) { + *out = *in + if in.Ingress != nil { + in, out := &in.Ingress, &out.Ingress + *out = new(IngressInfo) + (*in).DeepCopyInto(*out) + } + if in.HTTPRoute != nil { + in, out := &in.HTTPRoute, &out.HTTPRoute + *out = new(HTTPRouteInfo) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrafficRoutingInfo. +func (in *TrafficRoutingInfo) DeepCopy() *TrafficRoutingInfo { + if in == nil { + return nil + } + out := new(TrafficRoutingInfo) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadInfo) DeepCopyInto(out *WorkloadInfo) { + *out = *in + out.TypeMeta = in.TypeMeta + in.NameAndSpecData.DeepCopyInto(&out.NameAndSpecData) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadInfo. +func (in *WorkloadInfo) DeepCopy() *WorkloadInfo { + if in == nil { + return nil + } + out := new(WorkloadInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadRef) DeepCopyInto(out *WorkloadRef) { *out = *in diff --git a/config/crd/bases/rollouts.kruise.io_rollouthistories.yaml b/config/crd/bases/rollouts.kruise.io_rollouthistories.yaml new file mode 100644 index 0000000..23bd9c6 --- /dev/null +++ b/config/crd/bases/rollouts.kruise.io_rollouthistories.yaml @@ -0,0 +1,176 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: rollouthistories.rollouts.kruise.io +spec: + group: rollouts.kruise.io + names: + kind: RolloutHistory + listKind: RolloutHistoryList + plural: rollouthistories + singular: rollouthistory + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RolloutHistory is the Schema for the rollouthistories API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: RolloutHistorySpec defines the desired state of RolloutHistory + properties: + rollout: + description: Rollout indicates information of the rollout related + with rollouthistory + properties: + data: + description: Data indecates the spec of object ref + x-kubernetes-preserve-unknown-fields: true + name: + description: Name indicates the name of object ref, such as rollout + name, workload name, ingress name, etc. + type: string + rolloutID: + description: RolloutID indicates the new rollout if there is no + new RolloutID this time, ignore it and not execute RolloutHistory + type: string + required: + - name + - rolloutID + type: object + service: + description: Service indicates information of the service related + with workload + properties: + data: + description: Data indecates the spec of object ref + x-kubernetes-preserve-unknown-fields: true + name: + description: Name indicates the name of object ref, such as rollout + name, workload name, ingress name, etc. + type: string + required: + - name + type: object + trafficRouting: + description: TrafficRouting indicates information of traffic route + related with workload + properties: + httpRoute: + description: HTTPRouteRef indacates information of Gateway API + properties: + data: + description: Data indecates the spec of object ref + x-kubernetes-preserve-unknown-fields: true + name: + description: Name indicates the name of object ref, such as + rollout name, workload name, ingress name, etc. + type: string + required: + - name + type: object + ingress: + description: IngressRef indicates information of ingress + properties: + data: + description: Data indecates the spec of object ref + x-kubernetes-preserve-unknown-fields: true + name: + description: Name indicates the name of object ref, such as + rollout name, workload name, ingress name, etc. + type: string + required: + - name + type: object + type: object + workload: + description: Workload indicates information of the workload, such + as cloneset, deployment, advanced statefulset + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + data: + description: Data indecates the spec of object ref + x-kubernetes-preserve-unknown-fields: true + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: Name indicates the name of object ref, such as rollout + name, workload name, ingress name, etc. + type: string + required: + - name + type: object + type: object + status: + description: RolloutHistoryStatus defines the observed state of RolloutHistory + properties: + canarySteps: + description: CanarySteps indicates the pods released each step + items: + description: CanaryStepInfo indicates the pods for a revision + properties: + canaryStepIndex: + description: CanaryStepIndex indicates step this revision + format: int32 + type: integer + pods: + description: Pods indicates the pods information + items: + description: Pod indicates the information of a pod, including + name, ip, node_name. + properties: + ip: + description: IP indicates the pod ip + type: string + name: + description: Name indicates the node name + type: string + nodeName: + description: NodeName indicates the node which pod is + located at + type: string + type: object + type: array + type: object + type: array + phase: + description: Phase indicates phase of RolloutHistory, just "" or "completed" + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index a621998..c11d35d 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/rollouts.kruise.io_rollouts.yaml - bases/rollouts.kruise.io_batchreleases.yaml +- bases/rollouts.kruise.io_rollouthistories.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 97cb66a..cc69d72 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -262,6 +262,32 @@ rules: - get - patch - update +- apiGroups: + - rollouts.kruise.io + resources: + - rollouthistories + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rollouts.kruise.io + resources: + - rollouthistories/finalizers + verbs: + - update +- apiGroups: + - rollouts.kruise.io + resources: + - rollouthistories/status + verbs: + - get + - patch + - update - apiGroups: - rollouts.kruise.io resources: diff --git a/go.mod b/go.mod index f0e1bb9..d0aafe3 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,13 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.17.0 github.com/openkruise/kruise-api v1.0.0 + github.com/spf13/pflag v1.0.5 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.22.6 k8s.io/apiextensions-apiserver v0.22.6 k8s.io/apimachinery v0.22.6 k8s.io/client-go v0.22.6 + k8s.io/component-base v0.22.6 k8s.io/klog/v2 v2.10.0 k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e sigs.k8s.io/controller-runtime v0.10.3 diff --git a/go.sum b/go.sum index ca61d66..85dd4d9 100644 --- a/go.sum +++ b/go.sum @@ -404,8 +404,6 @@ github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/openkruise/kruise-api v1.0.0 h1:ScA0LxRRNBsgbcyLhTzR9B+KpGNWsIMptzzmjTqfYQo= github.com/openkruise/kruise-api v1.0.0/go.mod h1:kxV/UA/vrf/hz3z+kL21c0NOawC6K1ZjaKcJFgiOwsE= -github.com/openkruise/kruise-api v1.2.0 h1:MhoQtYT2tRdjrpb51xhn3lhEDWSlRGiMYQQ0Sh3zCkk= -github.com/openkruise/kruise-api v1.2.0/go.mod h1:BKMffjLFufZkj/yVpF5TjXG9gMU3Y9A3FxrVOJ5LJUI= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -976,7 +974,6 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.20.10/go.mod h1:0kei3F6biGjtRQBo5dUeujq6Ji3UCh9aOSfp/THYd7I= -k8s.io/api v0.20.15/go.mod h1:X3JDf1BiTRQQ6xNAxTuhgi6yL2dHc6fSr9LGzE+Z3YU= k8s.io/api v0.21.3/go.mod h1:hUgeYHUbBp23Ue4qdX9tR8/ANi/g3ehylAqDn9NWVOg= k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= @@ -987,7 +984,6 @@ k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQR k8s.io/apiextensions-apiserver v0.22.6 h1:TH+9+EGtoVzzbrlfSDnObzFTnyXKqw1NBfT5XFATeJI= k8s.io/apiextensions-apiserver v0.22.6/go.mod h1:wNsLwy8mfIkGThiv4Qq/Hy4qRazViKXqmH5pfYiRKyY= k8s.io/apimachinery v0.20.10/go.mod h1:kQa//VOAwyVwJ2+L9kOREbsnryfsGSkSM1przND4+mw= -k8s.io/apimachinery v0.20.15/go.mod h1:4KFiDSxCoGviCiRk9kTXIROsIf4VSGkVYjVJjJln3pg= k8s.io/apimachinery v0.21.3/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI= k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= @@ -997,14 +993,12 @@ k8s.io/apiserver v0.21.3/go.mod h1:eDPWlZG6/cCCMj/JBcEpDoK+I+6i3r9GsChYBHSbAzU= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= k8s.io/apiserver v0.22.6/go.mod h1:OlL1rGa2kKWGj2JEXnwBcul/BwC9Twe95gm4ohtiIIs= k8s.io/client-go v0.20.10/go.mod h1:fFg+aLoasv/R+xiVaWjxeqGFYltzgQcOQzkFaSRfnJ0= -k8s.io/client-go v0.20.15/go.mod h1:q/vywQFfGT3jw+lXQGA9sEJDH0QEX7XUT2PwrQ2qm/I= k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU= k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= k8s.io/client-go v0.22.6 h1:ugAXeC312xeGXsn7zTRz+btgtLBnW3qYhtUUpVQL7YE= k8s.io/client-go v0.22.6/go.mod h1:TffU4AV2idZGeP+g3kdFZP+oHVHWPL1JYFySOALriw0= k8s.io/code-generator v0.20.10/go.mod h1:i6FmG+QxaLxvJsezvZp0q/gAEzzOz3U53KFibghWToU= -k8s.io/code-generator v0.20.15/go.mod h1:MW85KuhTjX9nzhFYpRqUOYh4et0xeEBHTEjwBzFYGaM= k8s.io/code-generator v0.21.3/go.mod h1:K3y0Bv9Cz2cOW2vXUrNZlFbflhuPvuadW6JdnN6gGKo= k8s.io/code-generator v0.22.0/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= @@ -1031,8 +1025,6 @@ k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iL k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80= k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513 h1:pbudjNtv90nOgR0/DUhPwKHnQ55Khz8+sNhJBIK7A5M= -k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= diff --git a/main.go b/main.go index 85a720c..36b90c6 100644 --- a/main.go +++ b/main.go @@ -25,9 +25,12 @@ import ( rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" br "github.com/openkruise/rollouts/pkg/controller/batchrelease" "github.com/openkruise/rollouts/pkg/controller/rollout" + "github.com/openkruise/rollouts/pkg/controller/rollouthistory" "github.com/openkruise/rollouts/pkg/util" utilclient "github.com/openkruise/rollouts/pkg/util/client" + utilfeature "github.com/openkruise/rollouts/pkg/util/feature" "github.com/openkruise/rollouts/pkg/webhook" + "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -66,8 +69,10 @@ func main() { flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + utilfeature.DefaultMutableFeatureGate.AddFlag(pflag.CommandLine) klog.InitFlags(nil) - flag.Parse() + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() ctrl.SetLogger(klogr.New()) cfg := ctrl.GetConfigOrDie() @@ -109,6 +114,11 @@ func main() { os.Exit(1) } + if err = rollouthistory.Add(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "rollouthistory") + os.Exit(1) + } + //+kubebuilder:scaffold:builder setupLog.Info("setup webhook") if err = webhook.SetupWithManager(mgr); err != nil { diff --git a/pkg/controller/rollouthistory/rollouthistory_controller.go b/pkg/controller/rollouthistory/rollouthistory_controller.go new file mode 100644 index 0000000..be09159 --- /dev/null +++ b/pkg/controller/rollouthistory/rollouthistory_controller.go @@ -0,0 +1,395 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollouthistory + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "reflect" + + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/feature" + "github.com/openkruise/rollouts/pkg/util" + utilfeature "github.com/openkruise/rollouts/pkg/util/feature" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +var ( + concurrentReconciles = 3 +) + +func init() { + flag.IntVar(&concurrentReconciles, "rollouthistory-workers", 3, "Max concurrent workers for rolloutHistory controller.") +} + +// Add creates a new Rollout Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + if utilfeature.DefaultFeatureGate.Enabled(feature.RolloutHistoryGate) { + return add(mgr, newReconciler(mgr)) + } + return nil +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &RolloutHistoryReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Finder: newControllerFinder2(mgr.GetClient()), + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("rollouthistory-controller", mgr, controller.Options{ + Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + if err != nil { + return err + } + // Watch for changes to rollout + if err = c.Watch(&source.Kind{Type: &rolloutv1alpha1.Rollout{}}, &enqueueRequestForRollout{}); err != nil { + return err + } + // watch for changes to rolloutHistory + if err = c.Watch(&source.Kind{Type: &rolloutv1alpha1.RolloutHistory{}}, &enqueueRequestForRolloutHistory{}); err != nil { + return err + } + return nil +} + +// RolloutHistoryReconciler reconciles a RolloutHistory object +type RolloutHistoryReconciler struct { + client.Client + Scheme *runtime.Scheme + Finder *controllerFinder2 +} + +//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=rollouthistories,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=rollouthistories/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=rollouts.kruise.io,resources=rollouthistories/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the RolloutHistory object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.10.0/pkg/reconcile +func (r *RolloutHistoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // get rollout + rollout := &rolloutv1alpha1.Rollout{} + err := r.Get(ctx, req.NamespacedName, rollout) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + klog.Infof("Begin to reconcile Rollout %v", klog.KObj(rollout)) + + // ignore rollout without ObservedRolloutID + if rollout.Status.CanaryStatus == nil || + rollout.Status.CanaryStatus.ObservedRolloutID == "" { + return ctrl.Result{}, nil + } + // get RolloutHistory which is not completed and related to the rollout (only one or zero) + var rolloutHistory *rolloutv1alpha1.RolloutHistory + if rolloutHistory, err = r.getRolloutHistoryForRollout(rollout); err != nil { + klog.Errorf("get rollout(%s/%s) rolloutHistory(%s=%s) failed: %s", rollout.Namespace, rollout.Name, rolloutIDLabel, rollout.Status.CanaryStatus.ObservedRolloutID, err.Error()) + return ctrl.Result{}, err + } + // create a rolloutHistory when user does a new rollout + if rolloutHistory == nil { + // just create rolloutHistory for rollouts which are progressing, otherwise it's possible to create more than one rollouthistory when user does one rollout + if rollout.Status.Phase != rolloutv1alpha1.RolloutPhaseProgressing { + return ctrl.Result{}, nil + } + if err = r.createRolloutHistoryForProgressingRollout(rollout); err != nil { + klog.Errorf("create rollout(%s/%s) rolloutHistory(%s=%s) failed: %s", rollout.Namespace, rollout.Name, rolloutIDLabel, rollout.Status.CanaryStatus.ObservedRolloutID, err.Error()) + return ctrl.Result{}, err + } + klog.Infof("create rollout(%s/%s) rolloutHistory success", rollout.Namespace, rollout.Name) + return ctrl.Result{}, nil + } + + klog.Infof("get rollout(%s/%s) rolloutHistory(%s) success", rollout.Namespace, rollout.Name, rolloutHistory.Name) + + // update RolloutHistory which is waiting for rollout completed + if rolloutHistory.Status.Phase != rolloutv1alpha1.PhaseCompleted { + // update RolloutHistory when rollout .status.phase is equl to RolloutPhaseHealthy + if err = r.updateRolloutHistoryWhenRolloutIsCompeleted(rollout, rolloutHistory); err != nil { + klog.Errorf("update rollout(%s/%s) rolloutHistory(%s=%s) failed: %s", rollout.Namespace, rollout.Name, rolloutIDLabel, rollout.Status.CanaryStatus.ObservedRolloutID, err.Error()) + return ctrl.Result{}, err + } + // update rollouthistory success + klog.Infof("update rollout(%s/%s) rolloutHistory(%s) success", rollout.Namespace, rollout.Name, rolloutHistory.Name) + return ctrl.Result{}, nil + } + return ctrl.Result{}, nil +} + +// getRolloutHistoryForRollout get rolloutHistory according to rolloutID and rolloutName for this new rollout. +func (r *RolloutHistoryReconciler) getRolloutHistoryForRollout(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.RolloutHistory, error) { + // get labelSelector including rolloutBathID, rolloutID + lableSelectorString := fmt.Sprintf("%v=%v,%v=%v", rolloutIDLabel, rollout.Status.CanaryStatus.ObservedRolloutID, rolloutNameLabel, rollout.Name) + labelSelector, err := labels.Parse(lableSelectorString) + if err != nil { + return nil, err + } + // get rollouthistories according to labels, in fact there is only one or zero rolloutHistory with the labelSelector + rollouthistories := &rolloutv1alpha1.RolloutHistoryList{} + err = r.List(context.TODO(), rollouthistories, &client.ListOptions{LabelSelector: labelSelector}, client.InNamespace(rollout.Namespace)) + if err != nil { + return nil, err + } + // if there is no rollouthistory found, return + if len(rollouthistories.Items) == 0 { + return nil, nil + } + // find the rollouthistory + return &rollouthistories.Items[0], nil +} + +// createRolloutHistoryForProgressingRollout create a new rolloutHistory, which indicates that user does a new rollout +func (r *RolloutHistoryReconciler) createRolloutHistoryForProgressingRollout(rollout *rolloutv1alpha1.Rollout) error { + // init the rolloutHistory + rolloutHistory := &rolloutv1alpha1.RolloutHistory{ + ObjectMeta: v1.ObjectMeta{ + Name: rollout.Name + "-" + randAllString(6), + Namespace: rollout.Namespace, + Labels: map[string]string{ + rolloutIDLabel: rollout.Status.CanaryStatus.ObservedRolloutID, + rolloutNameLabel: rollout.Name, + }, + }, + } + // create the rolloutHistory + return r.Create(context.TODO(), rolloutHistory, &client.CreateOptions{}) +} + +// getRolloutHistorySpec get RolloutHistorySpec for rolloutHistory according to rollout +func (r *RolloutHistoryReconciler) getRolloutHistorySpec(rollout *rolloutv1alpha1.Rollout) (rolloutv1alpha1.RolloutHistorySpec, error) { + rolloutHistorySpec := rolloutv1alpha1.RolloutHistorySpec{} + var err error + // get rolloutInfo + if rolloutHistorySpec.Rollout, err = r.getRolloutInfo(rollout); err != nil { + return rolloutHistorySpec, err + } + // get workloadInfo + var workload *rolloutv1alpha1.WorkloadInfo + if workload, err = r.Finder.getWorkloadInfoForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef); err != nil { + return rolloutHistorySpec, err + } + rolloutHistorySpec.Workload = *workload + // get serviceInfo + if rolloutHistorySpec.Service, err = r.getServiceInfo(rollout); err != nil { + return rolloutHistorySpec, err + } + // get trafficRoutingInfo + if rolloutHistorySpec.TrafficRouting, err = r.getTrafficRoutingInfo(rollout); err != nil { + return rolloutHistorySpec, err + } + return rolloutHistorySpec, nil +} + +// getRolloutInfo get RolloutInfo to for rolloutHistorySpec +func (r *RolloutHistoryReconciler) getRolloutInfo(rollout *rolloutv1alpha1.Rollout) (rolloutv1alpha1.RolloutInfo, error) { + rolloutInfo := rolloutv1alpha1.RolloutInfo{} + var err error + rolloutInfo.Name = rollout.Name + rolloutInfo.RolloutID = rollout.Status.CanaryStatus.ObservedRolloutID + if rolloutInfo.Data.Raw, err = json.Marshal(rollout.Spec); err != nil { + return rolloutInfo, err + } + return rolloutInfo, nil +} + +// getServiceInfo get ServiceInfo for rolloutHistorySpec +func (r *RolloutHistoryReconciler) getServiceInfo(rollout *rolloutv1alpha1.Rollout) (rolloutv1alpha1.ServiceInfo, error) { + // get service + service := &corev1.Service{} + serviceInfo := rolloutv1alpha1.ServiceInfo{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service}, service) + if err != nil { + return serviceInfo, errors.New("service not find") + } + // marshal service into serviceInfo + serviceInfo.Name = service.Name + if serviceInfo.Data.Raw, err = json.Marshal(service.Spec); err != nil { + return serviceInfo, err + } + return serviceInfo, nil +} + +// getTrafficRoutingInfo get TrafficRoutingInfo for rolloutHistorySpec +func (r *RolloutHistoryReconciler) getTrafficRoutingInfo(rollout *rolloutv1alpha1.Rollout) (rolloutv1alpha1.TrafficRoutingInfo, error) { + trafficRoutingInfo := rolloutv1alpha1.TrafficRoutingInfo{} + var err error + // if gateway is configured, get it + if rollout.Spec.Strategy.Canary.TrafficRoutings[0].Gateway != nil && + rollout.Spec.Strategy.Canary.TrafficRoutings[0].Gateway.HTTPRouteName != nil { + trafficRoutingInfo.HTTPRoute, err = r.getGateWayInfo(rollout) + if err != nil { + return trafficRoutingInfo, err + } + } + // if ingress is configured, get it + if rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress != nil && + rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress.Name != "" { + trafficRoutingInfo.Ingress, err = r.getIngressInfo(rollout) + if err != nil { + return trafficRoutingInfo, err + } + } + return trafficRoutingInfo, nil +} + +// getGateWayInfo get HTTPRouteInfo for TrafficRoutingInfo +func (r *RolloutHistoryReconciler) getGateWayInfo(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.HTTPRouteInfo, error) { + // get HTTPRoute + gatewayName := *rollout.Spec.Strategy.Canary.TrafficRoutings[0].Gateway.HTTPRouteName + HTTPRoute := &v1alpha2.HTTPRoute{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: gatewayName}, HTTPRoute) + if err != nil { + return nil, errors.New("initGateway error: HTTPRoute " + gatewayName + " not find") + } + // marshal HTTPRoute into HTTPRouteInfo for rolloutHistory + gatewayInfo := &rolloutv1alpha1.HTTPRouteInfo{} + gatewayInfo.Name = HTTPRoute.Name + if gatewayInfo.Data.Raw, err = json.Marshal(HTTPRoute.Spec); err != nil { + return nil, err + } + return gatewayInfo, nil +} + +// getIngressInfo get IngressInfo for TrafficRoutingInfo +func (r *RolloutHistoryReconciler) getIngressInfo(rollout *rolloutv1alpha1.Rollout) (*rolloutv1alpha1.IngressInfo, error) { + // get Ingress + ingressName := rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress.Name + ingress := &networkingv1.Ingress{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: rollout.Namespace, Name: ingressName}, ingress) + if err != nil { + return nil, errors.New("initIngressInfo error: Ingress " + ingressName + " not find") + } + // marshal ingress into ingressInfo + ingressInfo := &rolloutv1alpha1.IngressInfo{} + ingressInfo.Name = ingressName + if ingressInfo.Data.Raw, err = json.Marshal(ingress.Spec); err != nil { + return nil, err + } + return ingressInfo, nil +} + +// updateRolloutHistoryWhenRolloutIsCompeleted record all pods released when rollout phase is healthy +func (r *RolloutHistoryReconciler) updateRolloutHistoryWhenRolloutIsCompeleted(rollout *rolloutv1alpha1.Rollout, rolloutHistory *rolloutv1alpha1.RolloutHistory) error { + // do update until rollout.status.Phase is equl to RolloutPhaseHealthy + if rollout.Status.Phase != rolloutv1alpha1.RolloutPhaseHealthy { + return nil + } + // when this rollot's phase has been healthy, rolloutHistory record steps information and rollout.spec + // record .spec for rolloutHistory + var err error + spec, err := r.getRolloutHistorySpec(rollout) + if err != nil { + return err + } + if !reflect.DeepEqual(rolloutHistory.Spec.Rollout.RolloutID, spec.Rollout.RolloutID) { + // update rolloutHistory Spec + rolloutHistory.Spec = spec + err = r.Update(context.TODO(), rolloutHistory, &client.UpdateOptions{}) + if err != nil { + return err + } + return nil + } + // make .status.phase PhaseCompleted + rolloutHistory.Status.Phase = rolloutv1alpha1.PhaseCompleted + // record all pods information for rolloutHistory .status.canarySteps + err = r.recordStatusCanarySteps(rollout, rolloutHistory) + if err != nil { + return err + } + // update rolloutHistory subresource + return r.Status().Update(context.TODO(), rolloutHistory, &client.UpdateOptions{}) +} + +// recordStatusCanarySteps record all pods information which are canary released +func (r *RolloutHistoryReconciler) recordStatusCanarySteps(rollout *rolloutv1alpha1.Rollout, rolloutHistory *rolloutv1alpha1.RolloutHistory) error { + rolloutHistory.Status.CanarySteps = make([]rolloutv1alpha1.CanaryStepInfo, 0) + for i := 0; i < len(rollout.Spec.Strategy.Canary.Steps); i++ { + podList := &corev1.PodList{} + var extraSelector labels.Selector + // get workload labelSelector + workloadLabelSelector, _ := r.Finder.getLabelSelectorForRef(rollout.Namespace, rollout.Spec.ObjectRef.WorkloadRef) + selector, err := v1.LabelSelectorAsSelector(workloadLabelSelector) + if err != nil { + return err + } + // get extra labelSelector including rolloutBathID, rolloutID and workload selector + lableSelectorString := fmt.Sprintf("%v=%v,%v=%v,%v", util.RolloutBatchIDLabel, len(rolloutHistory.Status.CanarySteps)+1, util.RolloutIDLabel, rolloutHistory.Spec.Rollout.RolloutID, selector.String()) + extraSelector, err = labels.Parse(lableSelectorString) + if err != nil { + return err + } + // get pods according to extra lableSelector + err = r.List(context.TODO(), podList, &client.ListOptions{LabelSelector: extraSelector}, client.InNamespace(rollout.Namespace)) + if err != nil { + return err + } + // if num of pods is empty, append a empty CanaryStepInfo{} to canarySteps + if len(podList.Items) == 0 { + index := int32(len(rolloutHistory.Status.CanarySteps)) + 1 + rolloutHistory.Status.CanarySteps = append(rolloutHistory.Status.CanarySteps, rolloutv1alpha1.CanaryStepInfo{CanaryStepIndex: index}) + continue + } + // get current step pods released + currentStepInfo := rolloutv1alpha1.CanaryStepInfo{} + var pods []rolloutv1alpha1.Pod + // get pods name, ip, node and add them to .status.canarySteps + for i := range podList.Items { + pod := &podList.Items[i] + if pod.DeletionTimestamp.IsZero() { + cur := rolloutv1alpha1.Pod{Name: pod.Name, IP: pod.Status.PodIP, NodeName: pod.Spec.NodeName} + pods = append(pods, cur) + } + } + currentStepInfo.Pods = pods + currentStepInfo.CanaryStepIndex = int32(len(rolloutHistory.Status.CanarySteps)) + 1 + // add current step pods to .status.canarySteps + rolloutHistory.Status.CanarySteps = append(rolloutHistory.Status.CanarySteps, currentStepInfo) + } + return nil +} diff --git a/pkg/controller/rollouthistory/rollouthistory_controller_test.go b/pkg/controller/rollouthistory/rollouthistory_controller_test.go new file mode 100644 index 0000000..eaad181 --- /dev/null +++ b/pkg/controller/rollouthistory/rollouthistory_controller_test.go @@ -0,0 +1,1062 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollouthistory + +import ( + "context" + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + utilpointer "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/gateway-api/apis/v1alpha2" + + "github.com/openkruise/kruise-api/apps/pub" + kruisev1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" +) + +func init() { + scheme = runtime.NewScheme() + _ = clientgoscheme.AddToScheme(scheme) + _ = kruisev1alpha1.AddToScheme(scheme) + _ = rolloutv1alpha1.AddToScheme(scheme) + _ = v1alpha2.AddToScheme(scheme) +} + +var ( + scheme *runtime.Scheme + + rollouthistoryDemo = rolloutv1alpha1.RolloutHistory{ + TypeMeta: metav1.TypeMeta{ + Kind: "RolloutHistory", + APIVersion: "rollouts.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rollouthistory-demo", + Namespace: "default", + Labels: map[string]string{ + rolloutIDLabel: "1", + rolloutNameLabel: "rollout-demo", + }, + }, + Spec: rolloutv1alpha1.RolloutHistorySpec{ + Rollout: rolloutv1alpha1.RolloutInfo{ + RolloutID: "1", + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "rollout-demo", + }, + }, + Service: rolloutv1alpha1.ServiceInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "service-demo", + }, + }, + TrafficRouting: rolloutv1alpha1.TrafficRoutingInfo{ + Ingress: &rolloutv1alpha1.IngressInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "ingress-demo", + }, + }, + HTTPRoute: &rolloutv1alpha1.HTTPRouteInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "HTTPRoute-demo", + }, + }, + }, + Workload: rolloutv1alpha1.WorkloadInfo{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + }, + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "workload-demo", + }, + }, + }, + Status: rolloutv1alpha1.RolloutHistoryStatus{ + Phase: "", + CanarySteps: []rolloutv1alpha1.CanaryStepInfo{ + { + CanaryStepIndex: 1, + Pods: []rolloutv1alpha1.Pod{ + { + Name: "pod-1", + IP: "1.2.3.4", + NodeName: "local", + }, + }, + }, + }, + }, + } + + rolloutDemo1 = rolloutv1alpha1.Rollout{ + TypeMeta: metav1.TypeMeta{ + Kind: "Rollout", + APIVersion: "rollouts.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rollout-demo", + Namespace: "default", + Labels: map[string]string{}, + }, + Spec: rolloutv1alpha1.RolloutSpec{ + ObjectRef: rolloutv1alpha1.ObjectRef{ + WorkloadRef: &rolloutv1alpha1.WorkloadRef{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + Name: "workload-demo", + }, + }, + RolloutID: "1", + Strategy: rolloutv1alpha1.RolloutStrategy{ + Canary: &rolloutv1alpha1.CanaryStrategy{ + Steps: []rolloutv1alpha1.CanaryStep{ + { + Weight: utilpointer.Int32(5), + Pause: rolloutv1alpha1.RolloutPause{ + Duration: utilpointer.Int32(0), + }, + }, + { + Weight: utilpointer.Int32(40), + Pause: rolloutv1alpha1.RolloutPause{}, + }, + { + Weight: utilpointer.Int32(100), + Pause: rolloutv1alpha1.RolloutPause{ + Duration: utilpointer.Int32(0), + }, + }, + }, + TrafficRoutings: []*rolloutv1alpha1.TrafficRouting{ + { + Service: "service-demo", + Ingress: &rolloutv1alpha1.IngressTrafficRouting{ + ClassType: "nginx", + Name: "ingress-demo", + }, + Gateway: &rolloutv1alpha1.GatewayTrafficRouting{ + HTTPRouteName: utilpointer.String("HTTPRoute-demo"), + }, + }, + }, + }, + }, + }, + } + + cloneSetDemo = kruisev1alpha1.CloneSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps.kruise.io/v1alpha1", + Kind: "CloneSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "workload-demo", + Namespace: "default", + Labels: map[string]string{ + "app": "echoserver", + }, + }, + Spec: kruisev1alpha1.CloneSetSpec{ + UpdateStrategy: kruisev1alpha1.CloneSetUpdateStrategy{ + Type: "InPlaceIfPossible", + InPlaceUpdateStrategy: &pub.InPlaceUpdateStrategy{ + GracePeriodSeconds: 1, + }, + }, + Replicas: utilpointer.Int32(5), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "echoserver", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "echoserver", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "echoserver", + Image: "cilium/echoserver:1.10.1", + ImagePullPolicy: "IfNotPresent", + Ports: []corev1.ContainerPort{ + { + ContainerPort: 8080, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "PORT", + Value: "8080", + }, + }, + }, + }, + }, + }, + }, + } + + serviceDemo = corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "service-demo", + Namespace: "default", + Labels: map[string]string{ + "app": "echoserver", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Port: int32(80), + TargetPort: intstr.IntOrString{IntVal: int32(8080)}, + Name: "http", + Protocol: "TCP", + }, + }, + Selector: map[string]string{ + "app": "echoserver", + }, + }, + } + + ingressDemo = networkingv1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "networking.k8s.io/v1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-demo", + Namespace: "default", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + }, + Spec: networkingv1.IngressSpec{ + Rules: []networkingv1.IngressRule{ + { + Host: "echoserver.example.com", + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{ + { + Path: "/apis/echo", + PathType: (*networkingv1.PathType)(utilpointer.String("Exact")), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: "service-demo", + Port: networkingv1.ServiceBackendPort{ + Number: int32(80), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + httpRouteDemo = v1alpha2.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "gateway.networking.k8s.io/v1alpha2", + Kind: "HTTPRoute", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "HTTPRoute-demo", + Namespace: "default", + }, + Spec: v1alpha2.HTTPRouteSpec{ + CommonRouteSpec: v1alpha2.CommonRouteSpec{ + ParentRefs: []v1alpha2.ParentRef{ + { + Name: "demo-lb", + }, + }, + }, + }, + } + + podDemo = corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-demo", + Namespace: "default", + Labels: map[string]string{ + util.RolloutBatchIDLabel: "1", + util.RolloutIDLabel: "1", + }, + }, + Spec: corev1.PodSpec{ + NodeName: "local", + }, + } +) + +func TestReconcile(t *testing.T) { + cases := []struct { + name string + req ctrl.Request + getPods func() []*corev1.Pod + getService func() []*corev1.Service + getWorkload func() []*kruisev1alpha1.CloneSet + getIngress func() []*networkingv1.Ingress + getHTTPRoute func() []*v1alpha2.HTTPRoute + getRollout func() []*rolloutv1alpha1.Rollout + getRolloutHistory func() []*rolloutv1alpha1.RolloutHistory + expectRolloutHistory func() []*rolloutv1alpha1.RolloutHistory + }{ + { + name: "test1, create a new rolloutHistory for rollout", + req: ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "rollout-demo", + }, + }, + getPods: func() []*corev1.Pod { + pods := []*corev1.Pod{} + return pods + }, + getService: func() []*corev1.Service { + services := []*corev1.Service{} + return services + }, + getWorkload: func() []*kruisev1alpha1.CloneSet { + workloads := []*kruisev1alpha1.CloneSet{} + return workloads + }, + getIngress: func() []*networkingv1.Ingress { + ingresses := []*networkingv1.Ingress{} + return ingresses + }, + getHTTPRoute: func() []*v1alpha2.HTTPRoute { + httpRoutes := []*v1alpha2.HTTPRoute{} + return httpRoutes + }, + getRollout: func() []*rolloutv1alpha1.Rollout { + rollout := rolloutDemo1.DeepCopy() + rollout.Status = rolloutv1alpha1.RolloutStatus{ + CanaryStatus: &rolloutv1alpha1.CanaryStatus{ + ObservedRolloutID: "1", + }, + Phase: rolloutv1alpha1.RolloutPhaseProgressing, + } + return []*rolloutv1alpha1.Rollout{rollout} + }, + getRolloutHistory: func() []*rolloutv1alpha1.RolloutHistory { + rollouthistories := []*rolloutv1alpha1.RolloutHistory{} + return rollouthistories + }, + expectRolloutHistory: func() []*rolloutv1alpha1.RolloutHistory { + rollouthistory := rollouthistoryDemo.DeepCopy() + rollouthistory.Spec = rolloutv1alpha1.RolloutHistorySpec{} + rollouthistory.Status = rolloutv1alpha1.RolloutHistoryStatus{} + return []*rolloutv1alpha1.RolloutHistory{rollouthistory} + }, + }, + { + name: "test2, completed a rolloutHistory for rollout", + req: ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "rollout-demo", + }, + }, + getPods: func() []*corev1.Pod { + pod1 := podDemo.DeepCopy() + pod1.Name = "pod1" + pod1.Spec.NodeName = "local" + pod1.Status = corev1.PodStatus{ + PodIP: "1.2.3.1", + } + pod1.Labels = map[string]string{ + util.RolloutBatchIDLabel: "1", + util.RolloutIDLabel: "2", + "app": "echoserver", + } + + pod2 := podDemo.DeepCopy() + pod2.Name = "pod2" + pod2.Spec.NodeName = "local" + pod2.Status = corev1.PodStatus{ + PodIP: "1.2.3.2", + } + pod2.Labels = map[string]string{ + util.RolloutBatchIDLabel: "2", + util.RolloutIDLabel: "2", + "app": "echoserver", + } + + pod3 := podDemo.DeepCopy() + pod3.Name = "pod3" + pod3.Spec.NodeName = "local" + pod3.Status = corev1.PodStatus{ + PodIP: "1.2.3.3", + } + pod3.Labels = map[string]string{ + util.RolloutBatchIDLabel: "3", + util.RolloutIDLabel: "2", + "app": "echoserver", + } + + pod4 := podDemo.DeepCopy() + pod4.Name = "pod4" + pod4.Spec.NodeName = "local" + pod4.Status = corev1.PodStatus{ + PodIP: "1.2.3.4", + } + pod4.Labels = map[string]string{ + util.RolloutBatchIDLabel: "3", + util.RolloutIDLabel: "2", + "app": "echoserver", + } + + pod5 := podDemo.DeepCopy() + pod5.Name = "pod5" + pod5.Spec.NodeName = "local" + pod5.Status = corev1.PodStatus{ + PodIP: "1.2.3.5", + } + pod5.Labels = map[string]string{ + util.RolloutBatchIDLabel: "3", + util.RolloutIDLabel: "2", + "app": "echoserver", + } + + return []*corev1.Pod{pod1, pod2, pod3, pod4, pod5} + }, + getService: func() []*corev1.Service { + services := []*corev1.Service{serviceDemo.DeepCopy()} + return services + }, + getWorkload: func() []*kruisev1alpha1.CloneSet { + workloads := []*kruisev1alpha1.CloneSet{cloneSetDemo.DeepCopy()} + return workloads + }, + getIngress: func() []*networkingv1.Ingress { + ingresses := []*networkingv1.Ingress{ingressDemo.DeepCopy()} + return ingresses + }, + getHTTPRoute: func() []*v1alpha2.HTTPRoute { + httpRoutes := []*v1alpha2.HTTPRoute{httpRouteDemo.DeepCopy()} + return httpRoutes + }, + getRollout: func() []*rolloutv1alpha1.Rollout { + rollout := rolloutDemo1.DeepCopy() + rollout.Spec.RolloutID = "2" + rollout.Status = rolloutv1alpha1.RolloutStatus{ + CanaryStatus: &rolloutv1alpha1.CanaryStatus{ + ObservedRolloutID: "2", + }, + Phase: rolloutv1alpha1.RolloutPhaseHealthy, + } + return []*rolloutv1alpha1.Rollout{rollout} + }, + getRolloutHistory: func() []*rolloutv1alpha1.RolloutHistory { + rollouthistory := rollouthistoryDemo.DeepCopy() + rollouthistory.Labels = map[string]string{ + rolloutIDLabel: "2", + rolloutNameLabel: "rollout-demo", + } + rollouthistory.Spec = rolloutv1alpha1.RolloutHistorySpec{} + rollouthistory.Status = rolloutv1alpha1.RolloutHistoryStatus{} + return []*rolloutv1alpha1.RolloutHistory{rollouthistory} + }, + expectRolloutHistory: func() []*rolloutv1alpha1.RolloutHistory { + rollouthistory := rollouthistoryDemo.DeepCopy() + rollouthistory.Labels = map[string]string{ + rolloutIDLabel: "2", + rolloutNameLabel: "rollout-demo", + } + rollouthistory.Spec = rolloutv1alpha1.RolloutHistorySpec{ + Rollout: rolloutv1alpha1.RolloutInfo{ + RolloutID: "2", + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "rollout-demo", + }, + }, + Workload: rolloutv1alpha1.WorkloadInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "workload-demo", + }, + }, + Service: rolloutv1alpha1.ServiceInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "service-demo", + }, + }, + TrafficRouting: rolloutv1alpha1.TrafficRoutingInfo{ + Ingress: &rolloutv1alpha1.IngressInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "ingress-demo", + }, + }, + HTTPRoute: &rolloutv1alpha1.HTTPRouteInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "HTTPRoute-demo", + }, + }, + }, + } + rollouthistory.Status = rolloutv1alpha1.RolloutHistoryStatus{ + Phase: rolloutv1alpha1.PhaseCompleted, + CanarySteps: []rolloutv1alpha1.CanaryStepInfo{ + { + CanaryStepIndex: 1, + Pods: []rolloutv1alpha1.Pod{ + { + Name: "pod1", + IP: "1.2.3.1", + NodeName: "local", + }, + }, + }, + { + CanaryStepIndex: 2, + Pods: []rolloutv1alpha1.Pod{ + { + Name: "pod2", + IP: "1.2.3.2", + NodeName: "local", + }, + }, + }, + { + CanaryStepIndex: 3, + Pods: []rolloutv1alpha1.Pod{ + { + Name: "pod3", + IP: "1.2.3.3", + NodeName: "local", + }, + { + Name: "pod4", + IP: "1.2.3.4", + NodeName: "local", + }, + { + Name: "pod5", + IP: "1.2.3.5", + NodeName: "local", + }, + }, + }, + }, + } + return []*rolloutv1alpha1.RolloutHistory{rollouthistory} + }, + }, + { + name: "test3, don't create a new rolloutHistory for rollout without rolloutID or ObservedRolloutID", + req: ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "rollout-demo", + }, + }, + getPods: func() []*corev1.Pod { + pods := []*corev1.Pod{} + return pods + }, + getService: func() []*corev1.Service { + services := []*corev1.Service{} + return services + }, + getWorkload: func() []*kruisev1alpha1.CloneSet { + workloads := []*kruisev1alpha1.CloneSet{} + return workloads + }, + getIngress: func() []*networkingv1.Ingress { + ingresses := []*networkingv1.Ingress{} + return ingresses + }, + getHTTPRoute: func() []*v1alpha2.HTTPRoute { + httpRoutes := []*v1alpha2.HTTPRoute{} + return httpRoutes + }, + getRollout: func() []*rolloutv1alpha1.Rollout { + rollout := rolloutDemo1.DeepCopy() + rollout.Spec.RolloutID = "" + rollout.Status = rolloutv1alpha1.RolloutStatus{ + CanaryStatus: &rolloutv1alpha1.CanaryStatus{ + ObservedRolloutID: "", + }, + Phase: rolloutv1alpha1.RolloutPhaseProgressing, + } + return []*rolloutv1alpha1.Rollout{rollout} + }, + getRolloutHistory: func() []*rolloutv1alpha1.RolloutHistory { + rollouthistories := []*rolloutv1alpha1.RolloutHistory{} + return rollouthistories + }, + expectRolloutHistory: func() []*rolloutv1alpha1.RolloutHistory { + rollouthistories := []*rolloutv1alpha1.RolloutHistory{} + return rollouthistories + }, + }, + { + name: "test4, don't create a new rolloutHistory for rollout which doesn't change its rolloutID", + req: ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "rollout-demo", + }, + }, + getPods: func() []*corev1.Pod { + pods := []*corev1.Pod{} + return pods + }, + getService: func() []*corev1.Service { + services := []*corev1.Service{} + return services + }, + getWorkload: func() []*kruisev1alpha1.CloneSet { + workloads := []*kruisev1alpha1.CloneSet{} + return workloads + }, + getIngress: func() []*networkingv1.Ingress { + ingresses := []*networkingv1.Ingress{} + return ingresses + }, + getHTTPRoute: func() []*v1alpha2.HTTPRoute { + httpRoutes := []*v1alpha2.HTTPRoute{} + return httpRoutes + }, + getRollout: func() []*rolloutv1alpha1.Rollout { + rollout := rolloutDemo1.DeepCopy() + rollout.Spec.RolloutID = "4" + rollout.Status = rolloutv1alpha1.RolloutStatus{ + CanaryStatus: &rolloutv1alpha1.CanaryStatus{ + ObservedRolloutID: "4", + }, + Phase: rolloutv1alpha1.RolloutPhaseProgressing, + } + return []*rolloutv1alpha1.Rollout{rollout} + }, + getRolloutHistory: func() []*rolloutv1alpha1.RolloutHistory { + rollouthistory := rollouthistoryDemo.DeepCopy() + rollouthistory.Labels = map[string]string{ + rolloutIDLabel: "4", + rolloutNameLabel: "rollout-demo", + } + rollouthistory.Spec = rolloutv1alpha1.RolloutHistorySpec{ + Rollout: rolloutv1alpha1.RolloutInfo{ + RolloutID: "4", + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "rollout-demo", + }, + }, + Workload: rolloutv1alpha1.WorkloadInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "workload-demo", + }, + }, + Service: rolloutv1alpha1.ServiceInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "service-demo", + }, + }, + TrafficRouting: rolloutv1alpha1.TrafficRoutingInfo{ + Ingress: &rolloutv1alpha1.IngressInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "ingress-demo", + }, + }, + HTTPRoute: &rolloutv1alpha1.HTTPRouteInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "HTTPRoute-demo", + }, + }, + }, + } + rollouthistory.Status = rolloutv1alpha1.RolloutHistoryStatus{ + Phase: rolloutv1alpha1.PhaseCompleted, + CanarySteps: []rolloutv1alpha1.CanaryStepInfo{ + { + CanaryStepIndex: 1, + Pods: []rolloutv1alpha1.Pod{ + { + Name: "pod1", + IP: "1.2.3.1", + NodeName: "local", + }, + }, + }, + { + CanaryStepIndex: 2, + Pods: []rolloutv1alpha1.Pod{ + { + Name: "pod2", + IP: "1.2.3.2", + NodeName: "local", + }, + }, + }, + { + CanaryStepIndex: 3, + Pods: []rolloutv1alpha1.Pod{ + { + Name: "pod3", + IP: "1.2.3.3", + NodeName: "local", + }, + { + Name: "pod4", + IP: "1.2.3.4", + NodeName: "local", + }, + { + Name: "pod5", + IP: "1.2.3.5", + NodeName: "local", + }, + }, + }, + }, + } + return []*rolloutv1alpha1.RolloutHistory{rollouthistory} + }, + expectRolloutHistory: func() []*rolloutv1alpha1.RolloutHistory { + rollouthistory := rollouthistoryDemo.DeepCopy() + rollouthistory.Labels = map[string]string{ + rolloutIDLabel: "4", + rolloutNameLabel: "rollout-demo", + } + rollouthistory.Spec = rolloutv1alpha1.RolloutHistorySpec{ + Rollout: rolloutv1alpha1.RolloutInfo{ + RolloutID: "4", + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "rollout-demo", + }, + }, + Workload: rolloutv1alpha1.WorkloadInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "workload-demo", + }, + }, + Service: rolloutv1alpha1.ServiceInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "service-demo", + }, + }, + TrafficRouting: rolloutv1alpha1.TrafficRoutingInfo{ + Ingress: &rolloutv1alpha1.IngressInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "ingress-demo", + }, + }, + HTTPRoute: &rolloutv1alpha1.HTTPRouteInfo{ + NameAndSpecData: rolloutv1alpha1.NameAndSpecData{ + Name: "HTTPRoute-demo", + }, + }, + }, + } + rollouthistory.Status = rolloutv1alpha1.RolloutHistoryStatus{ + Phase: rolloutv1alpha1.PhaseCompleted, + CanarySteps: []rolloutv1alpha1.CanaryStepInfo{ + { + CanaryStepIndex: 1, + Pods: []rolloutv1alpha1.Pod{ + { + Name: "pod1", + IP: "1.2.3.1", + NodeName: "local", + }, + }, + }, + { + CanaryStepIndex: 2, + Pods: []rolloutv1alpha1.Pod{ + { + Name: "pod2", + IP: "1.2.3.2", + NodeName: "local", + }, + }, + }, + { + CanaryStepIndex: 3, + Pods: []rolloutv1alpha1.Pod{ + { + Name: "pod3", + IP: "1.2.3.3", + NodeName: "local", + }, + { + Name: "pod4", + IP: "1.2.3.4", + NodeName: "local", + }, + { + Name: "pod5", + IP: "1.2.3.5", + NodeName: "local", + }, + }, + }, + }, + } + return []*rolloutv1alpha1.RolloutHistory{rollouthistory} + }, + }, + { + name: "test5, don't create a new rolloutHistory for rollout if its phase isn't RolloutPhaseProgressing", + req: ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "rollout-demo", + }, + }, + getPods: func() []*corev1.Pod { + pods := []*corev1.Pod{} + return pods + }, + getService: func() []*corev1.Service { + services := []*corev1.Service{} + return services + }, + getWorkload: func() []*kruisev1alpha1.CloneSet { + workloads := []*kruisev1alpha1.CloneSet{} + return workloads + }, + getIngress: func() []*networkingv1.Ingress { + ingresses := []*networkingv1.Ingress{} + return ingresses + }, + getHTTPRoute: func() []*v1alpha2.HTTPRoute { + httpRoutes := []*v1alpha2.HTTPRoute{} + return httpRoutes + }, + getRollout: func() []*rolloutv1alpha1.Rollout { + rollout := rolloutDemo1.DeepCopy() + rollout.Spec.RolloutID = "5" + rollout.Status = rolloutv1alpha1.RolloutStatus{ + CanaryStatus: &rolloutv1alpha1.CanaryStatus{ + ObservedRolloutID: "5", + }, + Phase: rolloutv1alpha1.RolloutPhaseHealthy, + } + return []*rolloutv1alpha1.Rollout{rollout} + }, + getRolloutHistory: func() []*rolloutv1alpha1.RolloutHistory { + rollouthistories := []*rolloutv1alpha1.RolloutHistory{} + return rollouthistories + }, + expectRolloutHistory: func() []*rolloutv1alpha1.RolloutHistory { + rollouthistories := []*rolloutv1alpha1.RolloutHistory{} + return rollouthistories + }, + }, + } + + for _, cs := range cases { + t.Run(cs.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + for _, obj := range cs.getPods() { + err := fakeClient.Create(context.TODO(), obj.DeepCopy(), &client.CreateOptions{}) + if err != nil { + t.Fatalf("create Pod failed: %s", err.Error()) + } + } + for _, obj := range cs.getRollout() { + err := fakeClient.Create(context.TODO(), obj.DeepCopy(), &client.CreateOptions{}) + if err != nil { + t.Fatalf("create Rollout failed: %s", err.Error()) + } + } + for _, obj := range cs.getWorkload() { + err := fakeClient.Create(context.TODO(), obj.DeepCopy(), &client.CreateOptions{}) + if err != nil { + t.Fatalf("create Workload failed: %s", err.Error()) + } + } + for _, obj := range cs.getService() { + err := fakeClient.Create(context.TODO(), obj.DeepCopy(), &client.CreateOptions{}) + if err != nil { + t.Fatalf("create Service failed: %s", err.Error()) + } + } + for _, obj := range cs.getIngress() { + err := fakeClient.Create(context.TODO(), obj.DeepCopy(), &client.CreateOptions{}) + if err != nil { + t.Fatalf("create Ingress failed: %s", err.Error()) + } + } + for _, obj := range cs.getHTTPRoute() { + err := fakeClient.Create(context.TODO(), obj.DeepCopy(), &client.CreateOptions{}) + if err != nil { + t.Fatalf("create HTTPRoute failed: %s", err.Error()) + } + } + for _, obj := range cs.getRolloutHistory() { + err := fakeClient.Create(context.TODO(), obj.DeepCopy(), &client.CreateOptions{}) + if err != nil { + t.Fatalf("create RolloutHistory failed: %s", err.Error()) + } + } + + recon := RolloutHistoryReconciler{ + Client: fakeClient, + Scheme: scheme, + Finder: newControllerFinder2(fakeClient), + } + + // Firstly, update spec, and requeue namespacedName + _, err := recon.Reconcile(context.TODO(), cs.req) + if err != nil { + t.Fatalf("Reconcile failed: %s", err.Error()) + } + + // Secondly, update spec, and requeue namespacedName + _, err = recon.Reconcile(context.TODO(), cs.req) + if err != nil { + t.Fatalf("Reconcile failed: %s", err.Error()) + } + + if !checkRolloutHistoryInfoEqual(fakeClient, t, cs.expectRolloutHistory()) { + t.Fatalf("Reconcile failed") + } + + if !checkRolloutHistoryNum(fakeClient, t, cs.expectRolloutHistory()) { + t.Fatalf("RolloutHistory generated invalid: %s", err.Error()) + } + }) + } +} + +func checkRolloutHistoryNum(c client.WithWatch, t *testing.T, expect []*rolloutv1alpha1.RolloutHistory) bool { + rollouthistories := &rolloutv1alpha1.RolloutHistoryList{} + err := c.List(context.TODO(), rollouthistories, &client.ListOptions{}, client.InNamespace("default")) + if err != nil { + t.Fatalf("get rollouthistories failed: %s", err.Error()) + } + if len(rollouthistories.Items) != len(expect) { + return false + } + return true +} + +func checkRolloutHistoryInfoEqual(c client.WithWatch, t *testing.T, expect []*rolloutv1alpha1.RolloutHistory) bool { + for i := range expect { + obj := expect[i] + rollouthistories := &rolloutv1alpha1.RolloutHistoryList{} + err := c.List(context.TODO(), rollouthistories, &client.ListOptions{}, client.InNamespace(obj.Namespace)) + if err != nil { + t.Fatalf("get rollouthistories failed: %s", err.Error()) + } + // in cases, there will be just one rollouthistory + if len(rollouthistories.Items) != 1 { + t.Fatalf("create rollouthistory failed: %s", err.Error()) + } + rollouthistory := rollouthistories.Items[0] + // compare Label + if !reflect.DeepEqual(obj.ObjectMeta.Labels, rollouthistory.ObjectMeta.Labels) { + t.Fatalf("diff rollouthistory label failed: %s", err.Error()) + return false + } + // compare Spec + if !checkRolloutHistorySpec(&obj.Spec, &rollouthistory.Spec) { + t.Fatalf("diff rollouthistory spec failed: %s", err.Error()) + return false + } + // compare status + // in the first reconcile, there is only spec updated + if !checkRolloutHistoryStatus(&obj.Status, &rollouthistory.Status) { + t.Fatalf("diff rollouthistory status failed: %s", err.Error()) + return false + } + } + + return true +} + +func checkRolloutHistorySpec(spec1 *rolloutv1alpha1.RolloutHistorySpec, spec2 *rolloutv1alpha1.RolloutHistorySpec) bool { + // spec1 and spec2 may be empty when rollouthistory is not completed + if reflect.DeepEqual(spec1, spec2) { + return true + } + // just compare those fields + if spec1.Rollout.Name != spec2.Rollout.Name || + spec1.Service.Name != spec2.Service.Name || + spec1.Workload.Name != spec2.Workload.Name || + spec1.Rollout.RolloutID != spec2.Rollout.RolloutID || + spec1.TrafficRouting.Ingress.Name != spec2.TrafficRouting.Ingress.Name || + spec1.TrafficRouting.HTTPRoute.Name != spec2.TrafficRouting.HTTPRoute.Name { + return false + } + + return true +} + +func checkRolloutHistoryStatus(status1 *rolloutv1alpha1.RolloutHistoryStatus, status2 *rolloutv1alpha1.RolloutHistoryStatus) bool { + // in the first reconcile, there is only spec updated + // status1 and status2 may be empty when rollouthistory is not completed + if reflect.DeepEqual(status1, status2) { + return true + } + // just compare those fields + if status1.Phase != status2.Phase || + len(status1.CanarySteps) != len(status2.CanarySteps) { + return false + } + // compare canarySteps, including CanaryStepIndex and pods for each canaryStep + for i := 0; i < len(status1.CanarySteps); i++ { + step1 := status1.CanarySteps[i] + step2 := status2.CanarySteps[i] + if step1.CanaryStepIndex != step2.CanaryStepIndex { + return false + } + if len(step1.Pods) != len(step2.Pods) { + return false + } + for j := 0; j < len(step1.Pods); j++ { + if step1.Pods[j].IP != step2.Pods[j].IP || + step1.Pods[j].Name != step2.Pods[j].Name || + step1.Pods[j].NodeName != step2.Pods[j].NodeName { + return false + } + } + } + + return true +} diff --git a/pkg/controller/rollouthistory/rollouthistory_event_handler.go b/pkg/controller/rollouthistory/rollouthistory_event_handler.go new file mode 100644 index 0000000..b8e8549 --- /dev/null +++ b/pkg/controller/rollouthistory/rollouthistory_event_handler.go @@ -0,0 +1,88 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollouthistory + +import ( + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var _ handler.EventHandler = &enqueueRequestForRolloutHistory{} + +type enqueueRequestForRolloutHistory struct { +} + +func (w *enqueueRequestForRolloutHistory) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { + w.handleEvent(q, evt.Object) +} + +func (w *enqueueRequestForRolloutHistory) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +} + +func (w *enqueueRequestForRolloutHistory) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +} + +func (w *enqueueRequestForRolloutHistory) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + w.handleEvent(q, evt.ObjectNew) +} + +func (w *enqueueRequestForRolloutHistory) handleEvent(q workqueue.RateLimitingInterface, obj client.Object) { + // In fact, rolloutHistory which is created by controller must have rolloutNameLabel and rolloutIDLabe + rolloutName, ok1 := obj.(*rolloutv1alpha1.RolloutHistory).Labels[rolloutNameLabel] + _, ok2 := obj.(*rolloutv1alpha1.RolloutHistory).Labels[rolloutIDLabel] + if !ok1 || !ok2 { + return + } + // add rollout which just creates a rolloutHistory to queue + nsn := types.NamespacedName{Namespace: obj.GetNamespace(), Name: rolloutName} + q.Add(reconcile.Request{NamespacedName: nsn}) +} + +var _ handler.EventHandler = &enqueueRequestForRollout{} + +type enqueueRequestForRollout struct { +} + +func (w *enqueueRequestForRollout) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { + w.handleEvent(q, evt.Object) +} + +func (w *enqueueRequestForRollout) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +} + +func (w *enqueueRequestForRollout) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +} + +func (w *enqueueRequestForRollout) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + w.handleEvent(q, evt.ObjectNew) +} + +func (w *enqueueRequestForRollout) handleEvent(q workqueue.RateLimitingInterface, obj client.Object) { + // RolloutID shouldn't be empty + rollout := obj.(*rolloutv1alpha1.Rollout) + if rollout.Status.CanaryStatus == nil || rollout.Status.CanaryStatus.ObservedRolloutID == "" { + return + } + // add rollout with RolloutID to queue + nsn := types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()} + q.Add(reconcile.Request{NamespacedName: nsn}) +} diff --git a/pkg/controller/rollouthistory/util.go b/pkg/controller/rollouthistory/util.go new file mode 100644 index 0000000..bc11849 --- /dev/null +++ b/pkg/controller/rollouthistory/util.go @@ -0,0 +1,342 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rollouthistory + +import ( + "context" + "crypto/rand" + "encoding/json" + "math/big" + "strings" + + appsv1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1" + appsv1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" + "github.com/openkruise/rollouts/pkg/util" + apps "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// For RolloutHistory +const ( + // rolloutIDLabel is designed to distinguish each rollout revision publications. + // The value of RolloutIDLabel corresponds Rollout.Spec.RolloutID. + rolloutIDLabel = "rollouts.kruise.io/rollout-id" + // rolloutName is a label key that will be patched to rolloutHistory. + // Only when rolloutIDLabel is set, rolloutNameLabel will be patched to rolloutHistory. + rolloutNameLabel = "rollouts.kruise.io/rollout-name" +) + +// controllerFinderFunc2 is a function type that maps to a rolloutHistory's WorkloadInfo +type controllerFinderFunc2 func(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*rolloutv1alpha1.WorkloadInfo, error) + +type controllerFinderFunc2LabelSelector func(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*v1.LabelSelector, error) +type controllerFinder2 struct { + client.Client +} + +func newControllerFinder2(c client.Client) *controllerFinder2 { + return &controllerFinder2{ + Client: c, + } +} + +func (r *controllerFinder2) getWorkloadInfoForRef(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*rolloutv1alpha1.WorkloadInfo, error) { + for _, finder := range r.finders2() { + workloadInfo, err := finder(namespace, ref) + if workloadInfo != nil || err != nil { + return workloadInfo, err + } + } + return nil, nil +} + +func (r *controllerFinder2) getLabelSelectorForRef(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*v1.LabelSelector, error) { + for _, finder := range r.finders2LabelSelector() { + labelSelector, err := finder(namespace, ref) + if labelSelector != nil || err != nil { + return labelSelector, err + } + } + return nil, nil +} + +func (r *controllerFinder2) finders2LabelSelector() []controllerFinderFunc2LabelSelector { + return []controllerFinderFunc2LabelSelector{r.getLabelSelectorWithAdvancedStatefulSet, r.getLabelSelectorWithCloneSet, + r.getLabelSelectorWithDeployment, r.getLabelSelectorWithNativeStatefulSet} +} + +func (r *controllerFinder2) finders2() []controllerFinderFunc2 { + return []controllerFinderFunc2{r.getWorkloadInfoWithAdvancedStatefulSet, r.getWorkloadInfoWithCloneSet, + r.getWorkloadInfoWithDeployment, r.getWorkloadInfoWithNativeStatefulSet} +} + +// getWorkloadInfoWithAdvancedStatefulSet returns WorkloadInfo with kruise statefulset referenced by the provided controllerRef +func (r *controllerFinder2) getWorkloadInfoWithAdvancedStatefulSet(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*rolloutv1alpha1.WorkloadInfo, error) { + // This error is irreversible, so there is no need to return error + ok, _ := verifyGroupKind(ref, util.ControllerKruiseKindSts.Kind, []string{util.ControllerKruiseKindSts.Group}) + if !ok { + return nil, nil + } + // get workload + workload := &appsv1beta1.StatefulSet{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: ref.Name}, workload) + if err != nil { + // when error is NotFound, it is ok here. + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + // assign value + workloadInfo := &rolloutv1alpha1.WorkloadInfo{} + workloadInfo.APIVersion = workload.APIVersion + workloadInfo.Kind = workload.Kind + workloadInfo.Name = workload.Name + if workloadInfo.Data.Raw, err = json.Marshal(workload.Spec); err != nil { + return nil, err + } + + return workloadInfo, nil +} + +// getLabelSelectorWithAdvancedStatefulSet returns selector with kruise statefulset referenced by the provided controllerRef +func (r *controllerFinder2) getLabelSelectorWithAdvancedStatefulSet(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*v1.LabelSelector, error) { + // This error is irreversible, so there is no need to return error + ok, _ := verifyGroupKind(ref, util.ControllerKruiseKindSts.Kind, []string{util.ControllerKruiseKindSts.Group}) + if !ok { + return nil, nil + } + // get workload + workload := &appsv1beta1.StatefulSet{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: ref.Name}, workload) + if err != nil { + // when error is NotFound, it is ok here. + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + // assign value + labelSelector := workload.Spec.Selector + + return labelSelector, nil +} + +// getWorkloadInfoWithCloneSet returns WorkloadInfo with kruise cloneset referenced by the provided controllerRef +func (r *controllerFinder2) getWorkloadInfoWithCloneSet(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*rolloutv1alpha1.WorkloadInfo, error) { + // This error is irreversible, so there is no need to return error + ok, _ := verifyGroupKind(ref, util.ControllerKruiseKindCS.Kind, []string{util.ControllerKruiseKindCS.Group}) + if !ok { + return nil, nil + } + // get workload + workload := &appsv1alpha1.CloneSet{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: ref.Name}, workload) + if err != nil { + // when error is NotFound, it is ok here. + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + // assign value + workloadInfo := &rolloutv1alpha1.WorkloadInfo{} + workloadInfo.APIVersion = workload.APIVersion + workloadInfo.Kind = workload.Kind + workloadInfo.Name = workload.Name + if workloadInfo.Data.Raw, err = json.Marshal(workload.Spec); err != nil { + return nil, err + } + + return workloadInfo, nil +} + +// getLabelSelectorWithCloneSet returns selector with kruise cloneset referenced by the provided controllerRef +func (r *controllerFinder2) getLabelSelectorWithCloneSet(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*v1.LabelSelector, error) { + // This error is irreversible, so there is no need to return error + ok, _ := verifyGroupKind(ref, util.ControllerKruiseKindCS.Kind, []string{util.ControllerKruiseKindCS.Group}) + if !ok { + return nil, nil + } + // get workload + workload := &appsv1alpha1.CloneSet{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: ref.Name}, workload) + if err != nil { + // when error is NotFound, it is ok here. + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + // assign value + labelSelector := workload.Spec.Selector + + return labelSelector, nil +} + +// getWorkloadInfoWithDeployment returns WorkloadInfo with k8s native deployment referenced by the provided controllerRef +func (r *controllerFinder2) getWorkloadInfoWithDeployment(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*rolloutv1alpha1.WorkloadInfo, error) { + // This error is irreversible, so there is no need to return error + ok, _ := verifyGroupKind(ref, util.ControllerKindDep.Kind, []string{util.ControllerKindDep.Group}) + if !ok { + return nil, nil + } + // get deployment + workload := &apps.Deployment{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: ref.Name}, workload) + if err != nil { + // when error is NotFound, it is ok here. + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + // assign value + workloadInfo := &rolloutv1alpha1.WorkloadInfo{} + workloadInfo.APIVersion = workload.APIVersion + workloadInfo.Kind = workload.Kind + workloadInfo.Name = workload.Name + if workloadInfo.Data.Raw, err = json.Marshal(workload.Spec); err != nil { + return nil, err + } + + return workloadInfo, nil +} + +// getLabelSelectorWithDeployment returns selector with deployment referenced by the provided controllerRef +func (r *controllerFinder2) getLabelSelectorWithDeployment(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*v1.LabelSelector, error) { + // This error is irreversible, so there is no need to return error + ok, _ := verifyGroupKind(ref, util.ControllerKindDep.Kind, []string{util.ControllerKindDep.Group}) + if !ok { + return nil, nil + } + // get workload + workload := &apps.Deployment{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: ref.Name}, workload) + if err != nil { + // when error is NotFound, it is ok here. + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + // assign value + labelSelector := workload.Spec.Selector + + return labelSelector, nil +} + +// getWorkloadInfoWithNativeStatefulSet returns WorkloadInfo with k8s native statefulset referenced by the provided controllerRef +func (r *controllerFinder2) getWorkloadInfoWithNativeStatefulSet(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*rolloutv1alpha1.WorkloadInfo, error) { + // This error is irreversible, so there is no need to return error + ok, _ := verifyGroupKind(ref, util.ControllerKindSts.Kind, []string{util.ControllerKindSts.Group}) + if !ok { + return nil, nil + } + // get workload + workload := &apps.StatefulSet{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: ref.Name}, workload) + if err != nil { + // when error is NotFound, it is ok here. + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + // assign value + workloadInfo := &rolloutv1alpha1.WorkloadInfo{} + workloadInfo.APIVersion = workload.APIVersion + workloadInfo.Kind = workload.Kind + workloadInfo.Name = workload.Name + if workloadInfo.Data.Raw, err = json.Marshal(workload.Spec); err != nil { + return nil, err + } + + return workloadInfo, nil +} + +// getLabelSelectorWithNativeStatefulSet returns selector with deployment referenced by the provided controllerRef +func (r *controllerFinder2) getLabelSelectorWithNativeStatefulSet(namespace string, ref *rolloutv1alpha1.WorkloadRef) (*v1.LabelSelector, error) { + // This error is irreversible, so there is no need to return error + ok, _ := verifyGroupKind(ref, util.ControllerKindSts.Kind, []string{util.ControllerKindSts.Group}) + if !ok { + return nil, nil + } + // get workload + workload := &apps.StatefulSet{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: ref.Name}, workload) + if err != nil { + // when error is NotFound, it is ok here. + if errors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + // assign value + labelSelector := workload.Spec.Selector + + return labelSelector, nil +} + +func verifyGroupKind(ref *rolloutv1alpha1.WorkloadRef, expectedKind string, expectedGroups []string) (bool, error) { + gv, err := schema.ParseGroupVersion(ref.APIVersion) + if err != nil { + return false, err + } + + if ref.Kind != expectedKind { + return false, nil + } + + for _, group := range expectedGroups { + if group == gv.Group { + return true, nil + } + } + + return false, nil +} + +var chars = []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", + "1", "2", "3", "4", "5", "6", "7", "8", "9", "0"} + +// randAllString get random string +func randAllString(lenNum int) string { + str := strings.Builder{} + for i := 0; i < lenNum; i++ { + n, err := rand.Int(rand.Reader, big.NewInt(36)) + if err != nil { + return "" + } + l := chars[n.Int64()] + str.WriteString(l) + } + return str.String() +} diff --git a/pkg/feature/rollout_features.go b/pkg/feature/rollout_features.go new file mode 100644 index 0000000..3eb9647 --- /dev/null +++ b/pkg/feature/rollout_features.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package feature + +import ( + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/component-base/featuregate" + + utilfeature "github.com/openkruise/rollouts/pkg/util/feature" +) + +const ( + // PodProbeMarkerGate enable Kruise provide the ability to execute custom Probes. + // Note: custom probe execution requires kruise daemon, so currently only traditional Kubelet is supported, not virtual-kubelet. + RolloutHistoryGate featuregate.Feature = "RolloutHistoryGate" +) + +var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + RolloutHistoryGate: {Default: false, PreRelease: featuregate.Alpha}, +} + +func init() { + runtime.Must(utilfeature.DefaultMutableFeatureGate.Add(defaultFeatureGates)) +} diff --git a/pkg/util/feature/feature_gate.go b/pkg/util/feature/feature_gate.go new file mode 100644 index 0000000..86d0625 --- /dev/null +++ b/pkg/util/feature/feature_gate.go @@ -0,0 +1,32 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package feature + +import ( + "k8s.io/component-base/featuregate" +) + +var ( + // DefaultMutableFeatureGate is a mutable version of DefaultFeatureGate. + // Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this. + // Tests that need to modify feature gates for the duration of their test should use: + // defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., )() + DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate() + + // DefaultFeatureGate is a shared global FeatureGate. + // Top-level commands/options setup that needs to modify this feature gate should use DefaultMutableFeatureGate. + DefaultFeatureGate featuregate.FeatureGate = DefaultMutableFeatureGate +)