Implementing a generic Ingress based on Lua And A/B Testing Release (#86)

* rollout support A/B Testing API

Signed-off-by: liheng.zms <liheng.zms@alibaba-inc.com>

* Implementing a generic Ingress based on Lua

Signed-off-by: liheng.zms <liheng.zms@alibaba-inc.com>

Signed-off-by: liheng.zms <liheng.zms@alibaba-inc.com>
This commit is contained in:
berg 2022-11-22 11:22:43 +08:00 committed by GitHub
parent 113527e6f3
commit 0c54037c60
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 2400 additions and 358 deletions

5
.lift.toml Normal file
View File

@ -0,0 +1,5 @@
# Ignore results from vendor directories
ignoreFiles = """
vendor/
lua_configuration/
"""

View File

@ -20,6 +20,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
FROM gcr.io/distroless/static:nonroot FROM gcr.io/distroless/static:nonroot
WORKDIR / WORKDIR /
COPY --from=builder /workspace/manager . COPY --from=builder /workspace/manager .
COPY lua_configuration /lua_configuration
USER 65532:65532 USER 65532:65532
ENTRYPOINT ["/manager"] ENTRYPOINT ["/manager"]

View File

@ -21,6 +21,7 @@ RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GOOS=linux GOARCH=amd64
FROM gcr.io/distroless/static:nonroot FROM gcr.io/distroless/static:nonroot
WORKDIR / WORKDIR /
COPY --from=builder /workspace/manager . COPY --from=builder /workspace/manager .
COPY lua_configuration /lua_configuration
USER 65532:65532 USER 65532:65532
ENTRYPOINT ["/manager"] ENTRYPOINT ["/manager"]

View File

@ -20,6 +20,7 @@ import (
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/intstr"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
) )
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
@ -65,11 +66,6 @@ type WorkloadRef struct {
Name string `json:"name"` Name string `json:"name"`
} }
/*type ControllerRevisionRef struct {
TargetRevisionName string `json:"targetRevisionName"`
SourceRevisionName string `json:"sourceRevisionName"`
}*/
// RolloutStrategy defines strategy to apply during next rollout // RolloutStrategy defines strategy to apply during next rollout
type RolloutStrategy struct { type RolloutStrategy struct {
// Paused indicates that the Rollout is paused. // Paused indicates that the Rollout is paused.
@ -116,7 +112,19 @@ type CanaryStep struct {
// Pause defines a pause stage for a rollout, manual or auto // Pause defines a pause stage for a rollout, manual or auto
// +optional // +optional
Pause RolloutPause `json:"pause,omitempty"` Pause RolloutPause `json:"pause,omitempty"`
// MetricsAnalysis *RolloutAnalysis `json:"metricsAnalysis,omitempty"` // Matches define conditions used for matching the incoming HTTP requests to canary service.
// Each match is independent, i.e. this rule will be matched if **any** one of the matches is satisfied.
// If Gateway API, current only support one match.
// And cannot support both weight and matches, if both are configured, then matches takes precedence.
Matches []HttpRouteMatch `json:"matches,omitempty"`
}
type HttpRouteMatch struct {
// Headers specifies HTTP request header matchers. Multiple match values are
// ANDed together, meaning, a request must match all the specified headers
// to select the route.
// +kubebuilder:validation:MaxItems=16
Headers []gatewayv1alpha2.HTTPHeaderMatch `json:"headers,omitempty"`
} }
// RolloutPause defines a pause stage for a rollout // RolloutPause defines a pause stage for a rollout
@ -141,7 +149,8 @@ type TrafficRouting struct {
// IngressTrafficRouting configuration for ingress controller to control traffic routing // IngressTrafficRouting configuration for ingress controller to control traffic routing
type IngressTrafficRouting struct { type IngressTrafficRouting struct {
// ClassType refers to the class type of an `Ingress`, e.g. Nginx. Default is Nginx // ClassType refers to the type of `Ingress`.
// current support nginx, aliyun-alb. default is nginx.
// +optional // +optional
ClassType string `json:"classType,omitempty"` ClassType string `json:"classType,omitempty"`
// Name refers to the name of an `Ingress` resource in the same namespace as the `Rollout` // Name refers to the name of an `Ingress` resource in the same namespace as the `Rollout`

View File

@ -24,6 +24,7 @@ package v1alpha1
import ( import (
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/gateway-api/apis/v1alpha2"
) )
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
@ -187,6 +188,13 @@ func (in *CanaryStep) DeepCopyInto(out *CanaryStep) {
**out = **in **out = **in
} }
in.Pause.DeepCopyInto(&out.Pause) in.Pause.DeepCopyInto(&out.Pause)
if in.Matches != nil {
in, out := &in.Matches, &out.Matches
*out = make([]HttpRouteMatch, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
} }
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryStep. // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryStep.
@ -293,6 +301,28 @@ func (in *HTTPRouteInfo) DeepCopy() *HTTPRouteInfo {
return out return out
} }
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *HttpRouteMatch) DeepCopyInto(out *HttpRouteMatch) {
*out = *in
if in.Headers != nil {
in, out := &in.Headers, &out.Headers
*out = make([]v1alpha2.HTTPHeaderMatch, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HttpRouteMatch.
func (in *HttpRouteMatch) DeepCopy() *HttpRouteMatch {
if in == nil {
return nil
}
out := new(HttpRouteMatch)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *IngressInfo) DeepCopyInto(out *IngressInfo) { func (in *IngressInfo) DeepCopyInto(out *IngressInfo) {
*out = *in *out = *in

View File

@ -108,6 +108,77 @@ spec:
items: items:
description: CanaryStep defines a step of a canary workload. description: CanaryStep defines a step of a canary workload.
properties: properties:
matches:
description: Matches define conditions used for matching
the incoming HTTP requests to canary service. Each
match is independent, i.e. this rule will be matched
if **any** one of the matches is satisfied. If Gateway
API, current only support one match. And cannot support
both weight and matches, if both are configured, then
matches takes precedence.
items:
properties:
headers:
description: Headers specifies HTTP request header
matchers. Multiple match values are ANDed together,
meaning, a request must match all the specified
headers to select the route.
items:
description: HTTPHeaderMatch describes how to
select a HTTP route by matching HTTP request
headers.
properties:
name:
description: "Name is the name of the HTTP
Header to be matched. Name matching MUST
be case insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).
\n If multiple entries specify equivalent
header names, only the first entry with
an equivalent name MUST be considered
for a match. Subsequent entries with an
equivalent header name MUST be ignored.
Due to the case-insensitivity of header
names, \"foo\" and \"Foo\" are considered
equivalent. \n When a header is repeated
in an HTTP request, it is implementation-specific
behavior as to how this is represented.
Generally, proxies should follow the guidance
from the RFC: https://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2
regarding processing a repeated header,
with special handling for \"Set-Cookie\"."
maxLength: 256
minLength: 1
pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$
type: string
type:
default: Exact
description: "Type specifies how to match
against the value of the header. \n Support:
Core (Exact) \n Support: Custom (RegularExpression)
\n Since RegularExpression HeaderMatchType
has custom conformance, implementations
can support POSIX, PCRE or any other dialects
of regular expressions. Please read the
implementation's documentation to determine
the supported dialect."
enum:
- Exact
- RegularExpression
type: string
value:
description: Value is the value of HTTP
Header to be matched.
maxLength: 4096
minLength: 1
type: string
required:
- name
- value
type: object
maxItems: 16
type: array
type: object
type: array
pause: pause:
description: Pause defines a pause stage for a rollout, description: Pause defines a pause stage for a rollout,
manual or auto manual or auto
@ -164,8 +235,9 @@ spec:
to route traffic, e.g. Nginx, Alb. to route traffic, e.g. Nginx, Alb.
properties: properties:
classType: classType:
description: ClassType refers to the class type description: ClassType refers to the type of `Ingress`.
of an `Ingress`, e.g. Nginx. Default is Nginx current support nginx, aliyun-alb. default is
nginx.
type: string type: string
name: name:
description: Name refers to the name of an `Ingress` description: Name refers to the name of an `Ingress`

View File

@ -144,6 +144,14 @@ rules:
- get - get
- patch - patch
- update - update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- apiGroups: - apiGroups:
- "" - ""
resources: resources:

3
go.mod
View File

@ -4,10 +4,12 @@ go 1.16
require ( require (
github.com/davecgh/go-spew v1.1.1 github.com/davecgh/go-spew v1.1.1
github.com/evanphx/json-patch v4.11.0+incompatible
github.com/onsi/ginkgo v1.16.5 github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.17.0 github.com/onsi/gomega v1.17.0
github.com/openkruise/kruise-api v1.0.0 github.com/openkruise/kruise-api v1.0.0
github.com/spf13/pflag v1.0.5 github.com/spf13/pflag v1.0.5
github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64
gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.22.6 k8s.io/api v0.22.6
k8s.io/apiextensions-apiserver v0.22.6 k8s.io/apiextensions-apiserver v0.22.6
@ -16,6 +18,7 @@ require (
k8s.io/component-base v0.22.6 k8s.io/component-base v0.22.6
k8s.io/klog/v2 v2.10.0 k8s.io/klog/v2 v2.10.0
k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e
layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf
sigs.k8s.io/controller-runtime v0.10.3 sigs.k8s.io/controller-runtime v0.10.3
sigs.k8s.io/gateway-api v0.4.3 sigs.k8s.io/gateway-api v0.4.3
sigs.k8s.io/yaml v1.2.0 sigs.k8s.io/yaml v1.2.0

5
go.sum
View File

@ -501,6 +501,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ=
github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
@ -675,6 +677,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -1031,6 +1034,8 @@ k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e h1:ldQh+neBabomh7+89dTpiFAB8tGdfVmuIzAHbvtl+9I= k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e h1:ldQh+neBabomh7+89dTpiFAB8tGdfVmuIzAHbvtl+9I=
k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf h1:rRz0YsF7VXj9fXRF6yQgFI7DzST+hsI3TeFSGupntu0=
layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf/go.mod h1:ivKkcY8Zxw5ba0jldhZCYYQfGdb2K6u9tbYK1AwMIBc=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

View File

@ -0,0 +1,35 @@
annotations = {}
if ( obj.annotations )
then
annotations = obj.annotations
end
annotations["alb.ingress.kubernetes.io/canary"] = "true"
annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = nil
annotations["alb.ingress.kubernetes.io/canary-weight"] = nil
if ( obj.weight ~= "-1" )
then
annotations["alb.ingress.kubernetes.io/canary-weight"] = obj.weight
end
if ( not obj.matches )
then
return annotations
end
for _,match in ipairs(obj.matches) do
local header = match.headers[1]
if ( header.name == "canary-by-cookie" )
then
annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = header.value
else
annotations["alb.ingress.kubernetes.io/canary-by-header"] = header.name
if ( header.type == "RegularExpression" )
then
annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
else
annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = header.value
end
end
end
return annotations

View File

@ -0,0 +1,46 @@
annotations = {}
-- obj.annotations is ingress annotations, it is recommended not to remove the part of the lua script, it must be kept
if ( obj.annotations )
then
annotations = obj.annotations
end
-- indicates the ingress is nginx canary api
annotations["nginx.ingress.kubernetes.io/canary"] = "true"
-- First, set all canary api to nil
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil
annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil
-- if rollout.spec.strategy.canary.steps.weight is nil, obj.weight will be -1,
-- then we need remove the canary-weight annotation
if ( obj.weight ~= "-1" )
then
annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight
end
-- if don't contains headers, immediate return annotations
if ( not obj.matches )
then
return annotations
end
-- headers & cookie apis
-- traverse matches
for _,match in ipairs(obj.matches) do
local header = match.headers[1]
-- cookie
if ( header.name == "canary-by-cookie" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
-- if regular expression
if ( header.type == "RegularExpression" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
end
end
end
-- must be return annotations
return annotations

View File

@ -74,6 +74,7 @@ type RolloutReconciler struct {
//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch //+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;update;patch //+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch
// Reconcile is part of the main kubernetes reconciliation loop which aims to // Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state. // move the current state of the cluster closer to the desired state.

View File

@ -99,6 +99,8 @@ func (r *RolloutReconciler) updateRolloutStatus(rollout *rolloutv1alpha1.Rollout
newStatus.Phase = rolloutv1alpha1.RolloutPhaseProgressing newStatus.Phase = rolloutv1alpha1.RolloutPhaseProgressing
cond := util.NewRolloutCondition(rolloutv1alpha1.RolloutConditionProgressing, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonInitializing, "Rollout is in Progressing") cond := util.NewRolloutCondition(rolloutv1alpha1.RolloutConditionProgressing, corev1.ConditionFalse, rolloutv1alpha1.ProgressingReasonInitializing, "Rollout is in Progressing")
util.SetRolloutCondition(&newStatus, *cond) util.SetRolloutCondition(&newStatus, *cond)
newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{}
newStatus.Message = "Rollout is in Progressing"
} else if newStatus.CanaryStatus == nil { } else if newStatus.CanaryStatus == nil {
// The following logic is to make PaaS be able to judge whether the rollout is ready // The following logic is to make PaaS be able to judge whether the rollout is ready
// at the first deployment of the Rollout/Workload. For example: generally, a PaaS // at the first deployment of the Rollout/Workload. For example: generally, a PaaS

View File

@ -24,7 +24,7 @@ import (
rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1" rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
"github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting" "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting"
"github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting/gateway" "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting/gateway"
"github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting/nginx" "github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting/ingress"
"github.com/openkruise/rollouts/pkg/util" "github.com/openkruise/rollouts/pkg/util"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/errors"
@ -40,9 +40,10 @@ func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) {
if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 {
return true, nil return true, nil
} }
// current only support one trafficRouting
if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { trafficRouting := r.rollout.Spec.Strategy.Canary.TrafficRoutings[0]
r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds if trafficRouting.GracePeriodSeconds <= 0 {
trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds
} }
canaryStatus := r.newStatus.CanaryStatus canaryStatus := r.newStatus.CanaryStatus
if r.newStatus.StableRevision == "" || canaryStatus.PodTemplateHash == "" { if r.newStatus.StableRevision == "" || canaryStatus.PodTemplateHash == "" {
@ -102,7 +103,7 @@ func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) {
} }
// After restore stable service configuration, give the ingress provider 3 seconds to take effect // After restore stable service configuration, give the ingress provider 3 seconds to take effect
if verifyTime := canaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { if verifyTime := canaryStatus.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) {
klog.Infof("update rollout(%s/%s) stable service(%s) done, and wait 3 seconds", r.rollout.Namespace, r.rollout.Name, r.stableService) klog.Infof("update rollout(%s/%s) stable service(%s) done, and wait 3 seconds", r.rollout.Namespace, r.rollout.Name, r.stableService)
return false, nil return false, nil
} }
@ -113,31 +114,30 @@ func (r *rolloutContext) doCanaryTrafficRouting() (bool, error) {
klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error())
return false, err return false, err
} }
var desiredWeight int32 currentStep := r.rollout.Spec.Strategy.Canary.Steps[canaryStatus.CurrentStepIndex-1]
if len(r.rollout.Spec.Strategy.Canary.Steps) > 0 { totalStep := len(r.rollout.Spec.Strategy.Canary.Steps)
desiredWeight = *r.rollout.Spec.Strategy.Canary.Steps[r.newStatus.CanaryStatus.CurrentStepIndex-1].Weight
}
steps := len(r.rollout.Spec.Strategy.Canary.Steps)
cond := util.GetRolloutCondition(*r.newStatus, rolloutv1alpha1.RolloutConditionProgressing) cond := util.GetRolloutCondition(*r.newStatus, rolloutv1alpha1.RolloutConditionProgressing)
cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and route traffic weight(%d)", canaryStatus.CurrentStepIndex, steps, desiredWeight) cond.Message = fmt.Sprintf("Rollout is in step(%d/%d), and route traffic (%d)", canaryStatus.CurrentStepIndex, totalStep, currentStep.Weight)
verify, err := trController.EnsureRoutes(context.TODO(), desiredWeight) verify, err := trController.EnsureRoutes(context.TODO(), currentStep.Weight, currentStep.Matches)
if err != nil { if err != nil {
return false, err return false, err
} else if !verify { } else if !verify {
klog.Infof("rollout(%s/%s) is doing step(%d) trafficRouting(%d)", r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex, desiredWeight) klog.Infof("rollout(%s/%s) is doing step(%d) trafficRouting(%s)", r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex, util.DumpJSON(currentStep))
return false, nil return false, nil
} }
klog.Infof("rollout(%s/%s) do step(%d) trafficRouting(%d) success", r.rollout.Namespace, r.rollout.Name, r.newStatus.CanaryStatus.CurrentStepIndex, desiredWeight) klog.Infof("rollout(%s/%s) do step(%d) trafficRouting(%s) success", r.rollout.Namespace, r.rollout.Name, canaryStatus.CurrentStepIndex, util.DumpJSON(currentStep))
return true, nil return true, nil
} }
// restore stable service configuration, remove selector pod-template-hash
func (r *rolloutContext) restoreStableService() (bool, error) { func (r *rolloutContext) restoreStableService() (bool, error) {
if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 {
return true, nil return true, nil
} }
// current only support one trafficRouting
if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { trafficRouting := r.rollout.Spec.Strategy.Canary.TrafficRoutings[0]
r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds if trafficRouting.GracePeriodSeconds <= 0 {
trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds
} }
//fetch stable service //fetch stable service
stableService := &corev1.Service{} stableService := &corev1.Service{}
@ -167,7 +167,7 @@ func (r *rolloutContext) restoreStableService() (bool, error) {
} }
// After restore stable service configuration, give the ingress provider 3 seconds to take effect // After restore stable service configuration, give the ingress provider 3 seconds to take effect
if r.newStatus.CanaryStatus.LastUpdateTime != nil { if r.newStatus.CanaryStatus.LastUpdateTime != nil {
if verifyTime := r.newStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds)); verifyTime.After(time.Now()) { if verifyTime := r.newStatus.CanaryStatus.LastUpdateTime.Add(time.Second * time.Duration(trafficRouting.GracePeriodSeconds)); verifyTime.After(time.Now()) {
klog.Infof("restore rollout(%s/%s) stable service(%s) done, and wait a moment", r.rollout.Namespace, r.rollout.Name, r.stableService) klog.Infof("restore rollout(%s/%s) stable service(%s) done, and wait a moment", r.rollout.Namespace, r.rollout.Name, r.stableService)
return false, nil return false, nil
} }
@ -176,13 +176,17 @@ func (r *rolloutContext) restoreStableService() (bool, error) {
return true, nil return true, nil
} }
// 1. restore ingress or gateway configuration
// 2. remove canary service
func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) { func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) {
if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 { if len(r.rollout.Spec.Strategy.Canary.TrafficRoutings) == 0 {
return true, nil return true, nil
} }
// current only support one trafficRouting
trafficRouting := r.rollout.Spec.Strategy.Canary.TrafficRoutings[0]
klog.Infof("rollout(%s/%s) start finalising traffic routing", r.rollout.Namespace, r.rollout.Name) klog.Infof("rollout(%s/%s) start finalising traffic routing", r.rollout.Namespace, r.rollout.Name)
if r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds <= 0 { if trafficRouting.GracePeriodSeconds <= 0 {
r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].GracePeriodSeconds = defaultGracePeriodSeconds trafficRouting.GracePeriodSeconds = defaultGracePeriodSeconds
} }
if r.newStatus.CanaryStatus == nil { if r.newStatus.CanaryStatus == nil {
r.newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{} r.newStatus.CanaryStatus = &rolloutv1alpha1.CanaryStatus{}
@ -193,7 +197,7 @@ func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) {
klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error()) klog.Errorf("rollout(%s/%s) newTrafficRoutingController failed: %s", r.rollout.Namespace, r.rollout.Name, err.Error())
return false, err return false, err
} }
verify, err := trController.EnsureRoutes(context.TODO(), 0) verify, err := trController.Finalise(context.TODO())
if err != nil { if err != nil {
return false, err return false, err
} else if !verify { } else if !verify {
@ -201,11 +205,6 @@ func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) {
return false, nil return false, nil
} }
// DoFinalising, such as delete nginx canary ingress
if err = trController.Finalise(context.TODO()); err != nil {
return false, err
}
// 2. remove canary service // 2. remove canary service
if r.newStatus.CanaryStatus.CanaryService == "" { if r.newStatus.CanaryStatus.CanaryService == "" {
return true, nil return true, nil
@ -226,20 +225,20 @@ func (r *rolloutContext) doFinalisingTrafficRouting() (bool, error) {
} }
func (r *rolloutContext) newTrafficRoutingController(roCtx *rolloutContext) (trafficrouting.Controller, error) { func (r *rolloutContext) newTrafficRoutingController(roCtx *rolloutContext) (trafficrouting.Controller, error) {
canary := roCtx.rollout.Spec.Strategy.Canary trafficRouting := roCtx.rollout.Spec.Strategy.Canary.TrafficRoutings[0]
if canary.TrafficRoutings[0].Ingress != nil { if trafficRouting.Ingress != nil {
gvk := schema.GroupVersionKind{Group: rolloutv1alpha1.GroupVersion.Group, Version: rolloutv1alpha1.GroupVersion.Version, Kind: "Rollout"} gvk := schema.GroupVersionKind{Group: rolloutv1alpha1.GroupVersion.Group, Version: rolloutv1alpha1.GroupVersion.Version, Kind: "Rollout"}
return nginx.NewNginxTrafficRouting(r.Client, r.newStatus, nginx.Config{ return ingress.NewIngressTrafficRouting(r.Client, ingress.Config{
RolloutName: r.rollout.Name, RolloutName: r.rollout.Name,
RolloutNs: r.rollout.Namespace, RolloutNs: r.rollout.Namespace,
CanaryService: r.canaryService, CanaryService: r.canaryService,
StableService: r.stableService, StableService: r.stableService,
TrafficConf: r.rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress, TrafficConf: trafficRouting.Ingress,
OwnerRef: *metav1.NewControllerRef(r.rollout, gvk), OwnerRef: *metav1.NewControllerRef(r.rollout, gvk),
}) })
} }
if canary.TrafficRoutings[0].Gateway != nil { if trafficRouting.Gateway != nil {
return gateway.NewGatewayTrafficRouting(r.Client, r.newStatus, gateway.Config{ return gateway.NewGatewayTrafficRouting(r.Client, gateway.Config{
RolloutName: r.rollout.Name, RolloutName: r.rollout.Name,
RolloutNs: r.rollout.Namespace, RolloutNs: r.rollout.Namespace,
CanaryService: r.canaryService, CanaryService: r.canaryService,

View File

@ -26,6 +26,7 @@ import (
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry" "k8s.io/client-go/util/retry"
"k8s.io/klog/v2" "k8s.io/klog/v2"
utilpointer "k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
) )
@ -40,17 +41,14 @@ type Config struct {
type gatewayController struct { type gatewayController struct {
client.Client client.Client
//stableIngress *netv1.Ingress conf Config
conf Config
newStatus *rolloutv1alpha1.RolloutStatus
} }
// NewGatewayTrafficRouting The Gateway API is a part of the SIG Network. // NewGatewayTrafficRouting The Gateway API is a part of the SIG Network.
func NewGatewayTrafficRouting(client client.Client, newStatus *rolloutv1alpha1.RolloutStatus, conf Config) (trafficrouting.Controller, error) { func NewGatewayTrafficRouting(client client.Client, conf Config) (trafficrouting.Controller, error) {
r := &gatewayController{ r := &gatewayController{
Client: client, Client: client,
conf: conf, conf: conf,
newStatus: newStatus,
} }
return r, nil return r, nil
} }
@ -60,25 +58,17 @@ func (r *gatewayController) Initialize(ctx context.Context) error {
return r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, route) return r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, route)
} }
func (r *gatewayController) EnsureRoutes(ctx context.Context, desiredWeight int32) (bool, error) { func (r *gatewayController) EnsureRoutes(ctx context.Context, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (bool, error) {
var httpRoute gatewayv1alpha2.HTTPRoute var httpRoute gatewayv1alpha2.HTTPRoute
err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, &httpRoute) err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, &httpRoute)
if err != nil { if err != nil {
// When desiredWeight=0, it means that rollout has been completed and the final traffic switching process is in progress,
// and The removal of httpRoute is expected.
if desiredWeight == 0 && errors.IsNotFound(err) {
klog.Infof("rollout(%s/%s) verify canary HTTPRoute has been deleted", r.conf.RolloutNs, r.conf.RolloutName)
return true, nil
}
return false, err return false, err
} }
// desired route // desired route
desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, desiredWeight) desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, weight, matches)
if reflect.DeepEqual(httpRoute.Spec.Rules, desiredRule) { if reflect.DeepEqual(httpRoute.Spec.Rules, desiredRule) {
return true, nil return true, nil
} }
// set route // set route
routeClone := &gatewayv1alpha2.HTTPRoute{} routeClone := &gatewayv1alpha2.HTTPRoute{}
if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
@ -92,24 +82,24 @@ func (r *gatewayController) EnsureRoutes(ctx context.Context, desiredWeight int3
klog.Errorf("update rollout(%s/%s) httpRoute(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, httpRoute.Name, err.Error()) klog.Errorf("update rollout(%s/%s) httpRoute(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, httpRoute.Name, err.Error())
return false, err return false, err
} }
klog.Infof("rollout(%s/%s) set HTTPRoute(name:%s weight:%d) success", r.conf.RolloutNs, r.conf.RolloutName, *r.conf.TrafficConf.HTTPRouteName, desiredWeight) klog.Infof("rollout(%s/%s) set HTTPRoute(name:%s weight:%d) success", r.conf.RolloutNs, r.conf.RolloutName, *r.conf.TrafficConf.HTTPRouteName, *weight)
return false, nil return false, nil
} }
func (r *gatewayController) Finalise(ctx context.Context) error { func (r *gatewayController) Finalise(ctx context.Context) (bool, error) {
httpRoute := &gatewayv1alpha2.HTTPRoute{} httpRoute := &gatewayv1alpha2.HTTPRoute{}
err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, httpRoute) err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: *r.conf.TrafficConf.HTTPRouteName}, httpRoute)
if err != nil { if err != nil {
if errors.IsNotFound(err) { if errors.IsNotFound(err) {
return nil return true, nil
} }
klog.Errorf("rollout(%s/%s) get HTTPRoute failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error()) klog.Errorf("rollout(%s/%s) get HTTPRoute failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
return err return false, err
} }
// desired rule // desired rule
desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, -1) desiredRule := r.buildDesiredHTTPRoute(httpRoute.Spec.Rules, utilpointer.Int32(-1), nil)
if reflect.DeepEqual(httpRoute.Spec.Rules, desiredRule) { if reflect.DeepEqual(httpRoute.Spec.Rules, desiredRule) {
return nil return true, nil
} }
routeClone := &gatewayv1alpha2.HTTPRoute{} routeClone := &gatewayv1alpha2.HTTPRoute{}
if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error { if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
@ -121,13 +111,68 @@ func (r *gatewayController) Finalise(ctx context.Context) error {
return r.Client.Update(context.TODO(), routeClone) return r.Client.Update(context.TODO(), routeClone)
}); err != nil { }); err != nil {
klog.Errorf("update rollout(%s/%s) httpRoute(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, httpRoute.Name, err.Error()) klog.Errorf("update rollout(%s/%s) httpRoute(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, httpRoute.Name, err.Error())
return err return false, err
} }
klog.Infof("rollout(%s/%s) TrafficRouting Finalise success", r.conf.RolloutNs, r.conf.RolloutName) klog.Infof("rollout(%s/%s) TrafficRouting Finalise success", r.conf.RolloutNs, r.conf.RolloutName)
return nil return false, nil
} }
func (r *gatewayController) buildDesiredHTTPRoute(rules []gatewayv1alpha2.HTTPRouteRule, canaryPercent int32) []gatewayv1alpha2.HTTPRouteRule { func (r *gatewayController) buildDesiredHTTPRoute(rules []gatewayv1alpha2.HTTPRouteRule, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) []gatewayv1alpha2.HTTPRouteRule {
var desired []gatewayv1alpha2.HTTPRouteRule
// Only when finalize method parameter weight=-1,
// then we need to remove the canary route policy and restore to the original configuration
if weight != nil && *weight == -1 {
for i := range rules {
rule := rules[i]
filterOutServiceBackendRef(&rule, r.conf.CanaryService)
_, stableRef := getServiceBackendRef(rule, r.conf.StableService)
if stableRef != nil {
stableRef.Weight = utilpointer.Int32(1)
setServiceBackendRef(&rule, *stableRef)
}
if len(rule.BackendRefs) != 0 {
desired = append(desired, rule)
}
}
return desired
// according to the Gateway API definition, weight and headers cannot be supported at the same time.
// A/B Testing, according to headers. current only support one match
} else if len(matches) > 0 {
return r.buildCanaryHeaderHttpRoutes(rules, matches[0].Headers)
}
// canary release, according to percentage of traffic routing
return r.buildCanaryWeightHttpRoutes(rules, weight)
}
func (r *gatewayController) buildCanaryHeaderHttpRoutes(rules []gatewayv1alpha2.HTTPRouteRule, headers []gatewayv1alpha2.HTTPHeaderMatch) []gatewayv1alpha2.HTTPRouteRule {
var desired []gatewayv1alpha2.HTTPRouteRule
var canarys []gatewayv1alpha2.HTTPRouteRule
for i := range rules {
rule := rules[i]
if _, canaryRef := getServiceBackendRef(rule, r.conf.CanaryService); canaryRef != nil {
continue
}
desired = append(desired, rule)
if _, stableRef := getServiceBackendRef(rule, r.conf.StableService); stableRef == nil {
continue
}
// according to stable rule to create canary rule
canaryRule := rule.DeepCopy()
_, canaryRef := getServiceBackendRef(*canaryRule, r.conf.StableService)
canaryRef.Name = gatewayv1alpha2.ObjectName(r.conf.CanaryService)
canaryRule.BackendRefs = []gatewayv1alpha2.HTTPBackendRef{*canaryRef}
// set canary headers in httpRoute
for j := range canaryRule.Matches {
match := &canaryRule.Matches[j]
match.Headers = append(match.Headers, headers...)
}
canarys = append(canarys, *canaryRule)
}
desired = append(desired, canarys...)
return desired
}
func (r *gatewayController) buildCanaryWeightHttpRoutes(rules []gatewayv1alpha2.HTTPRouteRule, weight *int32) []gatewayv1alpha2.HTTPRouteRule {
var desired []gatewayv1alpha2.HTTPRouteRule var desired []gatewayv1alpha2.HTTPRouteRule
for i := range rules { for i := range rules {
rule := rules[i] rule := rules[i]
@ -136,24 +181,16 @@ func (r *gatewayController) buildDesiredHTTPRoute(rules []gatewayv1alpha2.HTTPRo
desired = append(desired, rule) desired = append(desired, rule)
continue continue
} }
// If canaryPercent = -1, delete canary backendRef _, canaryRef := getServiceBackendRef(rule, r.conf.CanaryService)
if canaryPercent == -1 { if canaryRef == nil {
filterOutServiceBackendRef(&rule, r.conf.CanaryService) canaryRef = stableRef.DeepCopy()
stableRef.Weight = nil canaryRef.Name = gatewayv1alpha2.ObjectName(r.conf.CanaryService)
setServiceBackendRef(&rule, *stableRef)
// canaryPercent[0,100]
} else {
_, canaryRef := getServiceBackendRef(rule, r.conf.CanaryService)
if canaryRef == nil {
canaryRef = stableRef.DeepCopy()
canaryRef.Name = gatewayv1alpha2.ObjectName(r.conf.CanaryService)
}
stableWeight, canaryWeight := generateCanaryWeight(canaryPercent)
stableRef.Weight = &stableWeight
canaryRef.Weight = &canaryWeight
setServiceBackendRef(&rule, *stableRef)
setServiceBackendRef(&rule, *canaryRef)
} }
stableWeight, canaryWeight := generateCanaryWeight(*weight)
stableRef.Weight = &stableWeight
canaryRef.Weight = &canaryWeight
setServiceBackendRef(&rule, *stableRef)
setServiceBackendRef(&rule, *canaryRef)
desired = append(desired, rule) desired = append(desired, rule)
} }
return desired return desired

View File

@ -20,6 +20,8 @@ import (
"reflect" "reflect"
"testing" "testing"
rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
"github.com/openkruise/rollouts/pkg/util"
utilpointer "k8s.io/utils/pointer" utilpointer "k8s.io/utils/pointer"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
) )
@ -32,6 +34,13 @@ var (
Spec: gatewayv1alpha2.HTTPRouteSpec{ Spec: gatewayv1alpha2.HTTPRouteSpec{
Rules: []gatewayv1alpha2.HTTPRouteRule{ Rules: []gatewayv1alpha2.HTTPRouteRule{
{ {
Matches: []gatewayv1alpha2.HTTPRouteMatch{
{
Path: &gatewayv1alpha2.HTTPPathMatch{
Value: utilpointer.String("/web"),
},
},
},
BackendRefs: []gatewayv1alpha2.HTTPBackendRef{ BackendRefs: []gatewayv1alpha2.HTTPBackendRef{
{ {
BackendRef: gatewayv1alpha2.BackendRef{ BackendRef: gatewayv1alpha2.BackendRef{
@ -50,6 +59,17 @@ var (
Path: &gatewayv1alpha2.HTTPPathMatch{ Path: &gatewayv1alpha2.HTTPPathMatch{
Value: utilpointer.String("/store"), Value: utilpointer.String("/store"),
}, },
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "version",
Value: "v2",
},
},
},
{
Path: &gatewayv1alpha2.HTTPPathMatch{
Value: utilpointer.String("/v2/store"),
},
}, },
}, },
BackendRefs: []gatewayv1alpha2.HTTPBackendRef{ BackendRefs: []gatewayv1alpha2.HTTPBackendRef{
@ -88,7 +108,7 @@ var (
Matches: []gatewayv1alpha2.HTTPRouteMatch{ Matches: []gatewayv1alpha2.HTTPRouteMatch{
{ {
Path: &gatewayv1alpha2.HTTPPathMatch{ Path: &gatewayv1alpha2.HTTPPathMatch{
Value: utilpointer.String("/storev2"), Value: utilpointer.String("/storage"),
}, },
}, },
}, },
@ -113,67 +133,124 @@ func TestBuildDesiredHTTPRoute(t *testing.T) {
cases := []struct { cases := []struct {
name string name string
getRouteRules func() []gatewayv1alpha2.HTTPRouteRule getRouteRules func() []gatewayv1alpha2.HTTPRouteRule
canaryPercent int32 getRoutes func() (*int32, []rolloutsv1alpha1.HttpRouteMatch)
desiredRules func() []gatewayv1alpha2.HTTPRouteRule desiredRules func() []gatewayv1alpha2.HTTPRouteRule
}{ }{
{ {
name: "canary weight: 20", name: "test1 headers",
getRouteRules: func() []gatewayv1alpha2.HTTPRouteRule { getRouteRules: func() []gatewayv1alpha2.HTTPRouteRule {
rules := routeDemo.DeepCopy().Spec.Rules rules := routeDemo.DeepCopy().Spec.Rules
return rules return rules
}, },
canaryPercent: 20, getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
iType := gatewayv1alpha2.HeaderMatchRegularExpression
return nil, []rolloutsv1alpha1.HttpRouteMatch{
// header
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "user_id",
Value: "123*",
Type: &iType,
},
{
Name: "canary",
Value: "true",
},
},
},
}
},
desiredRules: func() []gatewayv1alpha2.HTTPRouteRule { desiredRules: func() []gatewayv1alpha2.HTTPRouteRule {
rules := routeDemo.DeepCopy().Spec.Rules rules := routeDemo.DeepCopy().Spec.Rules
rules[1].BackendRefs = []gatewayv1alpha2.HTTPBackendRef{ iType := gatewayv1alpha2.HeaderMatchRegularExpression
{ rules = append(rules, gatewayv1alpha2.HTTPRouteRule{
BackendRef: gatewayv1alpha2.BackendRef{ Matches: []gatewayv1alpha2.HTTPRouteMatch{
BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ {
Kind: &kindSvc, Path: &gatewayv1alpha2.HTTPPathMatch{
Name: "store-svc", Value: utilpointer.String("/store"),
Port: &portNum, },
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "version",
Value: "v2",
},
{
Name: "user_id",
Value: "123*",
Type: &iType,
},
{
Name: "canary",
Value: "true",
},
},
},
{
Path: &gatewayv1alpha2.HTTPPathMatch{
Value: utilpointer.String("/v2/store"),
},
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "user_id",
Value: "123*",
Type: &iType,
},
{
Name: "canary",
Value: "true",
},
}, },
Weight: utilpointer.Int32(80),
}, },
}, },
{ BackendRefs: []gatewayv1alpha2.HTTPBackendRef{
BackendRef: gatewayv1alpha2.BackendRef{ {
BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ BackendRef: gatewayv1alpha2.BackendRef{
Kind: &kindSvc, BackendObjectReference: gatewayv1alpha2.BackendObjectReference{
Name: "store-svc-canary", Kind: &kindSvc,
Port: &portNum, Name: "store-svc-canary",
Port: &portNum,
},
}, },
Weight: utilpointer.Int32(20),
}, },
}, },
} })
rules[3].BackendRefs = []gatewayv1alpha2.HTTPBackendRef{ rules = append(rules, gatewayv1alpha2.HTTPRouteRule{
{ Matches: []gatewayv1alpha2.HTTPRouteMatch{
BackendRef: gatewayv1alpha2.BackendRef{ {
BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ Path: &gatewayv1alpha2.HTTPPathMatch{
Kind: &kindSvc, Value: utilpointer.String("/storage"),
Name: "store-svc", },
Port: &portNum, Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "user_id",
Value: "123*",
Type: &iType,
},
{
Name: "canary",
Value: "true",
},
}, },
Weight: utilpointer.Int32(80),
}, },
}, },
{ BackendRefs: []gatewayv1alpha2.HTTPBackendRef{
BackendRef: gatewayv1alpha2.BackendRef{ {
BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ BackendRef: gatewayv1alpha2.BackendRef{
Kind: &kindSvc, BackendObjectReference: gatewayv1alpha2.BackendObjectReference{
Name: "store-svc-canary", Kind: &kindSvc,
Port: &portNum, Name: "store-svc-canary",
Port: &portNum,
},
}, },
Weight: utilpointer.Int32(20),
}, },
}, },
} })
return rules return rules
}, },
}, },
{ {
name: "canary weight: 0", name: "canary weight: 20",
getRouteRules: func() []gatewayv1alpha2.HTTPRouteRule { getRouteRules: func() []gatewayv1alpha2.HTTPRouteRule {
rules := routeDemo.DeepCopy().Spec.Rules rules := routeDemo.DeepCopy().Spec.Rules
rules[1].BackendRefs = []gatewayv1alpha2.HTTPBackendRef{ rules[1].BackendRefs = []gatewayv1alpha2.HTTPBackendRef{
@ -222,7 +299,9 @@ func TestBuildDesiredHTTPRoute(t *testing.T) {
} }
return rules return rules
}, },
canaryPercent: 0, getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
return utilpointer.Int32(20), nil
},
desiredRules: func() []gatewayv1alpha2.HTTPRouteRule { desiredRules: func() []gatewayv1alpha2.HTTPRouteRule {
rules := routeDemo.DeepCopy().Spec.Rules rules := routeDemo.DeepCopy().Spec.Rules
rules[1].BackendRefs = []gatewayv1alpha2.HTTPBackendRef{ rules[1].BackendRefs = []gatewayv1alpha2.HTTPBackendRef{
@ -233,7 +312,7 @@ func TestBuildDesiredHTTPRoute(t *testing.T) {
Name: "store-svc", Name: "store-svc",
Port: &portNum, Port: &portNum,
}, },
Weight: utilpointer.Int32(100), Weight: utilpointer.Int32(80),
}, },
}, },
{ {
@ -243,7 +322,7 @@ func TestBuildDesiredHTTPRoute(t *testing.T) {
Name: "store-svc-canary", Name: "store-svc-canary",
Port: &portNum, Port: &portNum,
}, },
Weight: utilpointer.Int32(0), Weight: utilpointer.Int32(20),
}, },
}, },
} }
@ -255,7 +334,7 @@ func TestBuildDesiredHTTPRoute(t *testing.T) {
Name: "store-svc", Name: "store-svc",
Port: &portNum, Port: &portNum,
}, },
Weight: utilpointer.Int32(100), Weight: utilpointer.Int32(80),
}, },
}, },
{ {
@ -265,7 +344,7 @@ func TestBuildDesiredHTTPRoute(t *testing.T) {
Name: "store-svc-canary", Name: "store-svc-canary",
Port: &portNum, Port: &portNum,
}, },
Weight: utilpointer.Int32(0), Weight: utilpointer.Int32(20),
}, },
}, },
} }
@ -320,11 +399,47 @@ func TestBuildDesiredHTTPRoute(t *testing.T) {
}, },
}, },
} }
iType := gatewayv1alpha2.HeaderMatchRegularExpression
rules = append(rules, gatewayv1alpha2.HTTPRouteRule{
Matches: []gatewayv1alpha2.HTTPRouteMatch{
{
Path: &gatewayv1alpha2.HTTPPathMatch{
Value: utilpointer.String("/storage"),
},
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "user_id",
Value: "123*",
Type: &iType,
},
{
Name: "canary",
Value: "true",
},
},
},
},
BackendRefs: []gatewayv1alpha2.HTTPBackendRef{
{
BackendRef: gatewayv1alpha2.BackendRef{
BackendObjectReference: gatewayv1alpha2.BackendObjectReference{
Kind: &kindSvc,
Name: "store-svc-canary",
Port: &portNum,
},
},
},
},
})
return rules return rules
}, },
canaryPercent: -1, getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
return utilpointer.Int32(-1), nil
},
desiredRules: func() []gatewayv1alpha2.HTTPRouteRule { desiredRules: func() []gatewayv1alpha2.HTTPRouteRule {
rules := routeDemo.DeepCopy().Spec.Rules rules := routeDemo.DeepCopy().Spec.Rules
rules[3].BackendRefs[0].Weight = utilpointer.Int32(1)
rules[1].BackendRefs[0].Weight = utilpointer.Int32(1)
return rules return rules
}, },
}, },
@ -339,9 +454,11 @@ func TestBuildDesiredHTTPRoute(t *testing.T) {
for _, cs := range cases { for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) { t.Run(cs.name, func(t *testing.T) {
controller := &gatewayController{conf: conf} controller := &gatewayController{conf: conf}
desired := controller.buildDesiredHTTPRoute(cs.getRouteRules(), cs.canaryPercent) weight, matches := cs.getRoutes()
if !reflect.DeepEqual(desired, cs.desiredRules()) { current := controller.buildDesiredHTTPRoute(cs.getRouteRules(), weight, matches)
t.Fatalf("expect: %v, but get %v", cs.desiredRules(), desired) desired := cs.desiredRules()
if !reflect.DeepEqual(current, desired) {
t.Fatalf("expect: %v, but get %v", util.DumpJSON(desired), util.DumpJSON(current))
} }
}) })
} }

View File

@ -0,0 +1,299 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ingress
import (
"context"
"encoding/json"
"fmt"
"reflect"
jsonpatch "github.com/evanphx/json-patch"
rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
"github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting"
"github.com/openkruise/rollouts/pkg/util"
"github.com/openkruise/rollouts/pkg/util/configuration"
"github.com/openkruise/rollouts/pkg/util/luamanager"
lua "github.com/yuin/gopher-lua"
netv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"
utilpointer "k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
)
type ingressController struct {
client.Client
conf Config
luaManager *luamanager.LuaManager
canaryIngressName string
luaScript string
}
type Config struct {
RolloutName string
RolloutNs string
CanaryService string
StableService string
TrafficConf *rolloutv1alpha1.IngressTrafficRouting
OwnerRef metav1.OwnerReference
}
func NewIngressTrafficRouting(client client.Client, conf Config) (trafficrouting.Controller, error) {
r := &ingressController{
Client: client,
conf: conf,
canaryIngressName: defaultCanaryIngressName(conf.TrafficConf.Name),
luaManager: &luamanager.LuaManager{},
}
luaScript, err := r.getTrafficRoutingIngressLuaScript(conf.TrafficConf.ClassType)
if err != nil {
return nil, err
}
r.luaScript = luaScript
return r, nil
}
// Initialize verify the existence of the ingress resource and generate the canary ingress
func (r *ingressController) Initialize(ctx context.Context) error {
ingress := &netv1.Ingress{}
err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.TrafficConf.Name}, ingress)
if err != nil {
return err
}
canaryIngress := &netv1.Ingress{}
err = r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.canaryIngressName}, canaryIngress)
if err != nil && !errors.IsNotFound(err) {
klog.Errorf("rollout(%s/%s) get canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, r.canaryIngressName, err.Error())
return err
} else if err == nil {
return nil
}
// build and create canary ingress
canaryIngress = r.buildCanaryIngress(ingress)
canaryIngress.Annotations, err = r.executeLuaForCanary(canaryIngress.Annotations, utilpointer.Int32(0), nil)
if err != nil {
klog.Errorf("rollout(%s/%s) execute lua failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
return err
}
if err = r.Create(ctx, canaryIngress); err != nil {
klog.Errorf("rollout(%s/%s) create canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
return err
}
klog.Infof("rollout(%s/%s) create canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, util.DumpJSON(canaryIngress))
return nil
}
func (r *ingressController) EnsureRoutes(ctx context.Context, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (bool, error) {
canaryIngress := &netv1.Ingress{}
err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: defaultCanaryIngressName(r.conf.TrafficConf.Name)}, canaryIngress)
if err != nil {
klog.Errorf("rollout(%s/%s) get canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
return false, err
}
newAnnotations, err := r.executeLuaForCanary(canaryIngress.Annotations, weight, matches)
if err != nil {
klog.Errorf("rollout(%s/%s) execute lua failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
return false, err
}
if reflect.DeepEqual(canaryIngress.Annotations, newAnnotations) {
return true, nil
}
byte1, _ := json.Marshal(metav1.ObjectMeta{Annotations: canaryIngress.Annotations})
byte2, _ := json.Marshal(metav1.ObjectMeta{Annotations: newAnnotations})
patch, err := jsonpatch.CreateMergePatch(byte1, byte2)
if err != nil {
klog.Errorf("rollout(%s/%s) create merge patch failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
return false, err
}
body := fmt.Sprintf(`{"metadata":%s}`, string(patch))
if err = r.Patch(ctx, canaryIngress, client.RawPatch(types.MergePatchType, []byte(body))); err != nil {
klog.Errorf("rollout(%s/%s) set canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error())
return false, err
}
klog.Infof("rollout(%s/%s) set canary ingress(%s) annotations(%s) success", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, util.DumpJSON(newAnnotations))
return false, nil
}
func (r *ingressController) Finalise(ctx context.Context) (bool, error) {
canaryIngress := &netv1.Ingress{}
err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.canaryIngressName}, canaryIngress)
if err != nil && !errors.IsNotFound(err) {
klog.Errorf("rollout(%s/%s) get canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, r.canaryIngressName, err.Error())
return false, err
}
if errors.IsNotFound(err) {
return true, nil
} else if !canaryIngress.DeletionTimestamp.IsZero() {
return false, nil
}
// First, set canary route 0 weight.
newAnnotations, err := r.executeLuaForCanary(canaryIngress.Annotations, utilpointer.Int32(0), nil)
if err != nil {
klog.Errorf("rollout(%s/%s) execute lua failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
return false, err
}
if !reflect.DeepEqual(canaryIngress.Annotations, newAnnotations) {
byte1, _ := json.Marshal(metav1.ObjectMeta{Annotations: canaryIngress.Annotations})
byte2, _ := json.Marshal(metav1.ObjectMeta{Annotations: newAnnotations})
patch, err := jsonpatch.CreateMergePatch(byte1, byte2)
if err != nil {
klog.Errorf("rollout(%s/%s) create merge patch failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
return false, err
}
body := fmt.Sprintf(`{"metadata":%s}`, string(patch))
if err = r.Patch(ctx, canaryIngress, client.RawPatch(types.MergePatchType, []byte(body))); err != nil {
klog.Errorf("rollout(%s/%s) set canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error())
return false, err
}
klog.Infof("rollout(%s/%s) set canary ingress annotations(%s) success", r.conf.RolloutNs, r.conf.RolloutName, util.DumpJSON(newAnnotations))
return false, nil
}
// Second, delete canary ingress
if err = r.Delete(ctx, canaryIngress); err != nil {
klog.Errorf("rollout(%s/%s) remove canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error())
return false, err
}
klog.Infof("rollout(%s/%s) remove canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name)
return false, nil
}
func (r *ingressController) buildCanaryIngress(stableIngress *netv1.Ingress) *netv1.Ingress {
desiredCanaryIngress := &netv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: r.canaryIngressName,
Namespace: stableIngress.Namespace,
Annotations: stableIngress.Annotations,
Labels: stableIngress.Labels,
},
Spec: netv1.IngressSpec{
Rules: make([]netv1.IngressRule, 0),
IngressClassName: stableIngress.Spec.IngressClassName,
TLS: stableIngress.Spec.TLS,
},
}
hosts := sets.NewString()
// Ensure canaryIngress is owned by this Rollout for cleanup
desiredCanaryIngress.SetOwnerReferences([]metav1.OwnerReference{r.conf.OwnerRef})
// Copy only the rules which reference the stableService from the stableIngress to the canaryIngress
// and change service backend to canaryService. Rules **not** referencing the stableIngress will be ignored.
for ir := 0; ir < len(stableIngress.Spec.Rules); ir++ {
var hasStableServiceBackendRule bool
stableRule := stableIngress.Spec.Rules[ir]
canaryRule := netv1.IngressRule{
Host: stableRule.Host,
IngressRuleValue: netv1.IngressRuleValue{
HTTP: &netv1.HTTPIngressRuleValue{},
},
}
// Update all backends pointing to the stableService to point to the canaryService now
for ip := 0; ip < len(stableRule.HTTP.Paths); ip++ {
if stableRule.HTTP.Paths[ip].Backend.Service.Name == r.conf.StableService {
hasStableServiceBackendRule = true
if stableRule.Host != "" {
hosts.Insert(stableRule.Host)
}
canaryPath := netv1.HTTPIngressPath{
Path: stableRule.HTTP.Paths[ip].Path,
PathType: stableRule.HTTP.Paths[ip].PathType,
Backend: stableRule.HTTP.Paths[ip].Backend,
}
canaryPath.Backend.Service.Name = r.conf.CanaryService
canaryRule.HTTP.Paths = append(canaryRule.HTTP.Paths, canaryPath)
}
}
// If this rule was using the specified stableService backend, append it to the canary Ingress spec
if hasStableServiceBackendRule {
desiredCanaryIngress.Spec.Rules = append(desiredCanaryIngress.Spec.Rules, canaryRule)
}
}
return desiredCanaryIngress
}
func defaultCanaryIngressName(name string) string {
return fmt.Sprintf("%s-canary", name)
}
func (r *ingressController) executeLuaForCanary(annotations map[string]string, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (map[string]string, error) {
if weight == nil {
// the lua script does not have a pointer type,
// so we need to pass weight=-1 to indicate the case where weight is nil.
weight = utilpointer.Int32(-1)
}
type LuaData struct {
Annotations map[string]string
Weight string
Matches []rolloutv1alpha1.HttpRouteMatch
CanaryService string
}
data := &LuaData{
Annotations: annotations,
Weight: fmt.Sprintf("%d", *weight),
Matches: matches,
CanaryService: r.conf.CanaryService,
}
unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data)
if err != nil {
return nil, err
}
obj := &unstructured.Unstructured{Object: unObj}
l, err := r.luaManager.RunLuaScript(obj, r.luaScript)
if err != nil {
return nil, err
}
returnValue := l.Get(-1)
if returnValue.Type() == lua.LTTable {
jsonBytes, err := luamanager.Encode(returnValue)
if err != nil {
return nil, err
}
newAnnotations := map[string]string{}
err = json.Unmarshal(jsonBytes, &newAnnotations)
if err != nil {
return nil, err
}
return newAnnotations, nil
}
return nil, fmt.Errorf("expect table output from Lua script, not %s", returnValue.Type().String())
}
func (r *ingressController) getTrafficRoutingIngressLuaScript(iType string) (string, error) {
if iType == "" {
iType = "nginx"
}
luaScript, err := configuration.GetTrafficRoutingIngressLuaScript(r.Client, iType)
if err != nil {
return "", err
}
if luaScript != "" {
return luaScript, nil
}
key := fmt.Sprintf("lua_configuration/trafficrouting_ingress/%s.lua", iType)
script := util.GetLuaConfigurationContent(key)
if script == "" {
return "", fmt.Errorf("%s lua script is not found", iType)
}
return script, nil
}

View File

@ -0,0 +1,626 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ingress
import (
"context"
"fmt"
"reflect"
"testing"
rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
"github.com/openkruise/rollouts/pkg/util"
"github.com/openkruise/rollouts/pkg/util/configuration"
corev1 "k8s.io/api/core/v1"
netv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
utilpointer "k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
)
var (
scheme *runtime.Scheme
demoConf = corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: configuration.RolloutConfigurationName,
Namespace: util.GetRolloutNamespace(),
},
Data: map[string]string{
fmt.Sprintf("%s.nginx", configuration.LuaTrafficRoutingIngressTypePrefix): `
annotations = obj.annotations
annotations["nginx.ingress.kubernetes.io/canary"] = "true"
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil
annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil
if ( obj.weight ~= "-1" )
then
annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight
end
if ( not obj.matches )
then
return annotations
end
for _,match in ipairs(obj.matches) do
header = match.headers[1]
if ( header.name == "canary-by-cookie" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
if ( header.type == "RegularExpression" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
end
end
end
return annotations
`,
fmt.Sprintf("%s.aliyun-alb", configuration.LuaTrafficRoutingIngressTypePrefix): `
function split(input, delimiter)
local arr = {}
string.gsub(input, '[^' .. delimiter ..']+', function(w) table.insert(arr, w) end)
return arr
end
annotations = obj.annotations
annotations["alb.ingress.kubernetes.io/canary"] = "true"
annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = nil
annotations["alb.ingress.kubernetes.io/canary-weight"] = nil
conditionKey = string.format("alb.ingress.kubernetes.io/conditions.%s", obj.canaryService)
annotations[conditionKey] = nil
if ( obj.weight ~= "-1" )
then
annotations["alb.ingress.kubernetes.io/canary-weight"] = obj.weight
end
if ( not obj.matches )
then
return annotations
end
conditions = {}
match = obj.matches[1]
for _,header in ipairs(match.headers) do
condition = {}
if ( header.name == "Cookie" )
then
condition.type = "Cookie"
condition.cookieConfig = {}
cookies = split(header.value, ";")
values = {}
for _,cookieStr in ipairs(cookies) do
cookie = split(cookieStr, "=")
value = {}
value.key = cookie[1]
value.value = cookie[2]
table.insert(values, value)
end
condition.cookieConfig.values = values
elseif ( header.name == "SourceIp" )
then
condition.type = "SourceIp"
condition.sourceIpConfig = {}
ips = split(header.value, ";")
values = {}
for _,ip in ipairs(ips) do
table.insert(values, ip)
end
condition.sourceIpConfig.values = values
else
condition.type = "Header"
condition.headerConfig = {}
condition.headerConfig.key = header.name
vals = split(header.value, ";")
values = {}
for _,val in ipairs(vals) do
table.insert(values, val)
end
condition.headerConfig.values = values
end
table.insert(conditions, condition)
end
annotations[conditionKey] = json.encode(conditions)
return annotations
`,
},
}
demoIngress = netv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "echoserver",
Annotations: map[string]string{
"kubernetes.io/ingress.class": "nginx",
},
},
Spec: netv1.IngressSpec{
IngressClassName: utilpointer.String("nginx"),
TLS: []netv1.IngressTLS{
{
Hosts: []string{"echoserver.example.com"},
SecretName: "echoserver-name",
},
{
Hosts: []string{"log.example.com"},
SecretName: "log-name",
},
},
Rules: []netv1.IngressRule{
{
Host: "echoserver.example.com",
IngressRuleValue: netv1.IngressRuleValue{
HTTP: &netv1.HTTPIngressRuleValue{
Paths: []netv1.HTTPIngressPath{
{
Path: "/apis/echo",
Backend: netv1.IngressBackend{
Service: &netv1.IngressServiceBackend{
Name: "echoserver",
Port: netv1.ServiceBackendPort{
Number: 80,
},
},
},
},
{
Path: "/apis/other",
Backend: netv1.IngressBackend{
Service: &netv1.IngressServiceBackend{
Name: "other",
Port: netv1.ServiceBackendPort{
Number: 80,
},
},
},
},
},
},
},
},
{
Host: "log.example.com",
IngressRuleValue: netv1.IngressRuleValue{
HTTP: &netv1.HTTPIngressRuleValue{
Paths: []netv1.HTTPIngressPath{
{
Path: "/apis/logs",
Backend: netv1.IngressBackend{
Service: &netv1.IngressServiceBackend{
Name: "echoserver",
Port: netv1.ServiceBackendPort{
Number: 8899,
},
},
},
},
},
},
},
},
},
},
}
)
func init() {
scheme = runtime.NewScheme()
_ = clientgoscheme.AddToScheme(scheme)
_ = rolloutsv1alpha1.AddToScheme(scheme)
}
func TestInitialize(t *testing.T) {
cases := []struct {
name string
getConfigmap func() *corev1.ConfigMap
getIngress func() []*netv1.Ingress
expectIngress func() *netv1.Ingress
}{
{
name: "init test1",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
return []*netv1.Ingress{demoIngress.DeepCopy()}
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0"
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
}
config := Config{
RolloutName: "rollout-demo",
StableService: "echoserver",
CanaryService: "echoserver-canary",
TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{
Name: "echoserver",
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build()
fakeCli.Create(context.TODO(), cs.getConfigmap())
for _, ingress := range cs.getIngress() {
fakeCli.Create(context.TODO(), ingress)
}
controller, err := NewIngressTrafficRouting(fakeCli, config)
if err != nil {
t.Fatalf("NewIngressTrafficRouting failed: %s", err.Error())
return
}
err = controller.Initialize(context.TODO())
if err != nil {
t.Fatalf("Initialize failed: %s", err.Error())
return
}
canaryIngress := &netv1.Ingress{}
err = fakeCli.Get(context.TODO(), client.ObjectKey{Name: "echoserver-canary"}, canaryIngress)
if err != nil {
t.Fatalf("Get canary ingress failed: %s", err.Error())
return
}
expect := cs.expectIngress()
if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) ||
!reflect.DeepEqual(canaryIngress.Spec, expect.Spec) {
t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress))
}
})
}
}
func TestEnsureRoutes(t *testing.T) {
cases := []struct {
name string
getConfigmap func() *corev1.ConfigMap
getIngress func() []*netv1.Ingress
getRoutes func() (*int32, []rolloutsv1alpha1.HttpRouteMatch)
expectIngress func() *netv1.Ingress
ingressType string
}{
{
name: "ensure routes test1",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0"
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
return nil, []rolloutsv1alpha1.HttpRouteMatch{
// header
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "user_id",
Value: "123456",
},
},
},
// cookies
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "canary-by-cookie",
Value: "demo",
},
},
},
}
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo"
expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id"
expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456"
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
{
name: "ensure routes test2",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456"
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
return utilpointer.Int32(40), nil
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "40"
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
{
name: "ensure routes test3",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456"
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
iType := gatewayv1alpha2.HeaderMatchRegularExpression
return nil, []rolloutsv1alpha1.HttpRouteMatch{
// header
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "user_id",
Value: "123*",
Type: &iType,
},
},
},
}
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id"
expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = "123*"
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
{
name: "ensure routes test4",
ingressType: "aliyun-alb",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["alb.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["alb.ingress.kubernetes.io/canary-weight"] = "0"
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
return nil, []rolloutsv1alpha1.HttpRouteMatch{
// header
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "Cookie",
Value: "demo1=value1;demo2=value2",
},
{
Name: "SourceIp",
Value: "192.168.0.0/16;172.16.0.0/16",
},
{
Name: "headername",
Value: "headervalue1;headervalue2",
},
},
},
}
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["alb.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["alb.ingress.kubernetes.io/conditions.echoserver-canary"] = `[{"cookieConfig":{"values":[{"key":"demo1","value":"value1"},{"key":"demo2","value":"value2"}]},"type":"Cookie"},{"sourceIpConfig":{"values":["192.168.0.0/16","172.16.0.0/16"]},"type":"SourceIp"},{"headerConfig":{"key":"headername","values":["headervalue1","headervalue2"]},"type":"Header"}]`
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
}
config := Config{
RolloutName: "rollout-demo",
StableService: "echoserver",
CanaryService: "echoserver-canary",
TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{
Name: "echoserver",
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build()
fakeCli.Create(context.TODO(), cs.getConfigmap())
for _, ingress := range cs.getIngress() {
fakeCli.Create(context.TODO(), ingress)
}
config.TrafficConf.ClassType = cs.ingressType
controller, err := NewIngressTrafficRouting(fakeCli, config)
if err != nil {
t.Fatalf("NewIngressTrafficRouting failed: %s", err.Error())
return
}
weight, matches := cs.getRoutes()
_, err = controller.EnsureRoutes(context.TODO(), weight, matches)
if err != nil {
t.Fatalf("EnsureRoutes failed: %s", err.Error())
return
}
canaryIngress := &netv1.Ingress{}
err = fakeCli.Get(context.TODO(), client.ObjectKey{Name: "echoserver-canary"}, canaryIngress)
if err != nil {
t.Fatalf("Get canary ingress failed: %s", err.Error())
return
}
expect := cs.expectIngress()
if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) ||
!reflect.DeepEqual(canaryIngress.Spec, expect.Spec) {
t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress))
}
})
}
}
func TestFinalise(t *testing.T) {
cases := []struct {
name string
getConfigmap func() *corev1.ConfigMap
getIngress func() []*netv1.Ingress
expectIngress func() *netv1.Ingress
}{
{
name: "finalise test1",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456"
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0"
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
{
name: "finalise test2",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0"
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
expectIngress: func() *netv1.Ingress {
return nil
},
},
}
config := Config{
RolloutName: "rollout-demo",
StableService: "echoserver",
CanaryService: "echoserver-canary",
TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{
Name: "echoserver",
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build()
fakeCli.Create(context.TODO(), cs.getConfigmap())
for _, ingress := range cs.getIngress() {
fakeCli.Create(context.TODO(), ingress)
}
controller, err := NewIngressTrafficRouting(fakeCli, config)
if err != nil {
t.Fatalf("NewIngressTrafficRouting failed: %s", err.Error())
return
}
_, err = controller.Finalise(context.TODO())
if err != nil {
t.Fatalf("EnsureRoutes failed: %s", err.Error())
return
}
canaryIngress := &netv1.Ingress{}
err = fakeCli.Get(context.TODO(), client.ObjectKey{Name: "echoserver-canary"}, canaryIngress)
if err != nil {
if cs.expectIngress() == nil && errors.IsNotFound(err) {
return
}
t.Fatalf("Get canary ingress failed: %s", err.Error())
return
}
expect := cs.expectIngress()
if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) ||
!reflect.DeepEqual(canaryIngress.Spec, expect.Spec) {
t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress))
}
})
}
}

View File

@ -16,20 +16,28 @@ limitations under the License.
package trafficrouting package trafficrouting
import "context" import (
"context"
rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
)
// Controller common function across all TrafficRouting implementation // Controller common function across all TrafficRouting implementation
type Controller interface { type Controller interface {
// Initialize will validate the traffic routing resource // Initialize will validate the traffic routing resource
// 1. Ingress type, verify the existence of the ingress resource and generate the canary ingress[weight=0%]
// 2. Gateway type, verify the existence of the gateway resource
Initialize(ctx context.Context) error Initialize(ctx context.Context) error
// EnsureRoutes check and set canary desired weight. desiredWeight[0,100] // EnsureRoutes check and set canary weight and matches.
// weight indicates percentage of traffic to canary service, and range of values[0,100]
// matches indicates A/B Testing release for headers, cookies
// 1. check if canary has been set desired weight. // 1. check if canary has been set desired weight.
// 2. If not, set canary desired weight // 2. If not, set canary desired weight
// When the first set weight is returned false, mainly to give the provider some time to process, only when again ensure, will return true // When the first set weight is returned false, mainly to give the provider some time to process, only when again ensure, will return true
// When desiredWeight=0, it means that rollout has been completed and the final traffic switching process is in progress, EnsureRoutes(ctx context.Context, weight *int32, matches []rolloutv1alpha1.HttpRouteMatch) (bool, error)
// and the canary weight should be set to 0 at this time.
EnsureRoutes(ctx context.Context, desiredWeight int32) (bool, error)
// Finalise will do some cleanup work after the canary rollout complete, such as delete canary ingress. // Finalise will do some cleanup work after the canary rollout complete, such as delete canary ingress.
// Finalise is called with a 3-second delay after completing the canary. // Finalise is called with a 3-second delay after completing the canary.
Finalise(ctx context.Context) error // bool indicates whether function Finalise is complete,
// for example, when ingress type, only canary ingress Not Found is considered function finalise complete
Finalise(ctx context.Context) (bool, error)
} }

View File

@ -1,205 +0,0 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nginx
import (
"context"
"fmt"
"strconv"
rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
"github.com/openkruise/rollouts/pkg/controller/rollout/trafficrouting"
"github.com/openkruise/rollouts/pkg/util"
netv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
)
const (
nginxIngressAnnotationDefaultPrefix = "nginx.ingress.kubernetes.io"
k8sIngressClassAnnotation = "kubernetes.io/ingress.class"
)
type nginxController struct {
client.Client
//stableIngress *netv1.Ingress
conf Config
newStatus *rolloutv1alpha1.RolloutStatus
}
type Config struct {
RolloutName string
RolloutNs string
CanaryService string
StableService string
TrafficConf *rolloutv1alpha1.IngressTrafficRouting
OwnerRef metav1.OwnerReference
}
func NewNginxTrafficRouting(client client.Client, newStatus *rolloutv1alpha1.RolloutStatus, conf Config) (trafficrouting.Controller, error) {
r := &nginxController{
Client: client,
conf: conf,
newStatus: newStatus,
}
return r, nil
}
func (r *nginxController) Initialize(ctx context.Context) error {
ingress := &netv1.Ingress{}
return r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.TrafficConf.Name}, ingress)
}
func (r *nginxController) EnsureRoutes(ctx context.Context, desiredWeight int32) (bool, error) {
canaryIngress := &netv1.Ingress{}
err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.defaultCanaryIngressName()}, canaryIngress)
if err != nil && !errors.IsNotFound(err) {
klog.Errorf("rollout(%s/%s) get canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
return false, err
}
// When desiredWeight=0, it means that rollout has been completed and the final traffic switching process is in progress,
// and the canary weight should be set to 0 at this time.
if desiredWeight == 0 {
// canary ingress
if errors.IsNotFound(err) || !canaryIngress.DeletionTimestamp.IsZero() {
klog.Infof("rollout(%s/%s) verify canary ingress has been deleted", r.conf.RolloutNs, r.conf.RolloutName)
return true, nil
}
} else if errors.IsNotFound(err) {
// create canary ingress
stableIngress := &netv1.Ingress{}
err = r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.TrafficConf.Name}, stableIngress)
if err != nil {
return false, err
} else if !stableIngress.DeletionTimestamp.IsZero() {
klog.Warningf("rollout(%s/%s) stable ingress is deleting", r.conf.RolloutNs, r.conf.RolloutName)
return false, nil
}
canaryIngress = r.buildCanaryIngress(stableIngress, desiredWeight)
if err = r.Create(ctx, canaryIngress); err != nil {
klog.Errorf("rollout(%s/%s) create canary ingress failed: %s", r.conf.RolloutNs, r.conf.RolloutName, err.Error())
return false, err
}
data := util.DumpJSON(canaryIngress)
klog.Infof("rollout(%s/%s) create canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, data)
return false, nil
}
// check whether canary weight equals desired weight
currentWeight := getIngressCanaryWeight(canaryIngress)
if desiredWeight == currentWeight {
return true, nil
}
cloneObj := canaryIngress.DeepCopy()
body := fmt.Sprintf(`{"metadata":{"annotations":{"%s/canary-weight":"%s"}}}`, nginxIngressAnnotationDefaultPrefix, fmt.Sprintf("%d", desiredWeight))
if err = r.Patch(ctx, cloneObj, client.RawPatch(types.StrategicMergePatchType, []byte(body))); err != nil {
klog.Errorf("rollout(%s/%s) set canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error())
return false, err
}
klog.Infof("rollout(%s/%s) set ingress routes(weight:%d) success", r.conf.RolloutNs, r.conf.RolloutName, desiredWeight)
return false, nil
}
func (r *nginxController) Finalise(ctx context.Context) error {
canaryIngress := &netv1.Ingress{}
err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.defaultCanaryIngressName()}, canaryIngress)
if err != nil && !errors.IsNotFound(err) {
klog.Errorf("rollout(%s/%s) get canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, r.defaultCanaryIngressName(), err.Error())
return err
}
if errors.IsNotFound(err) || !canaryIngress.DeletionTimestamp.IsZero() {
return nil
}
// immediate delete canary ingress
if err = r.Delete(ctx, canaryIngress); err != nil {
klog.Errorf("rollout(%s/%s) remove canary ingress(%s) failed: %s", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name, err.Error())
return err
}
klog.Infof("rollout(%s/%s) remove canary ingress(%s) success", r.conf.RolloutNs, r.conf.RolloutName, canaryIngress.Name)
return nil
}
func (r *nginxController) buildCanaryIngress(stableIngress *netv1.Ingress, desiredWeight int32) *netv1.Ingress {
desiredCanaryIngress := &netv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: r.defaultCanaryIngressName(),
Namespace: stableIngress.Namespace,
Annotations: map[string]string{},
},
Spec: netv1.IngressSpec{
Rules: make([]netv1.IngressRule, 0),
},
}
// Preserve ingressClassName from stable ingress
if stableIngress.Spec.IngressClassName != nil {
desiredCanaryIngress.Spec.IngressClassName = stableIngress.Spec.IngressClassName
}
// Must preserve ingress.class on canary ingress, no other annotations matter
// See: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#canary
if val, ok := stableIngress.Annotations[k8sIngressClassAnnotation]; ok {
desiredCanaryIngress.Annotations[k8sIngressClassAnnotation] = val
}
// Ensure canaryIngress is owned by this Rollout for cleanup
desiredCanaryIngress.SetOwnerReferences([]metav1.OwnerReference{r.conf.OwnerRef})
// Copy only the rules which reference the stableService from the stableIngress to the canaryIngress
// and change service backend to canaryService. Rules **not** referencing the stableIngress will be ignored.
for ir := 0; ir < len(stableIngress.Spec.Rules); ir++ {
var hasStableServiceBackendRule bool
ingressRule := stableIngress.Spec.Rules[ir].DeepCopy()
// Update all backends pointing to the stableService to point to the canaryService now
for ip := 0; ip < len(ingressRule.HTTP.Paths); ip++ {
if ingressRule.HTTP.Paths[ip].Backend.Service.Name == r.conf.StableService {
hasStableServiceBackendRule = true
ingressRule.HTTP.Paths[ip].Backend.Service.Name = r.conf.CanaryService
}
}
// If this rule was using the specified stableService backend, append it to the canary Ingress spec
if hasStableServiceBackendRule {
desiredCanaryIngress.Spec.Rules = append(desiredCanaryIngress.Spec.Rules, *ingressRule)
}
}
// Always set `canary` and `canary-weight` - `canary-by-header` and `canary-by-cookie`, if set, will always take precedence
desiredCanaryIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)] = "true"
desiredCanaryIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)] = fmt.Sprintf("%d", desiredWeight)
return desiredCanaryIngress
}
func (r *nginxController) defaultCanaryIngressName() string {
return fmt.Sprintf("%s-canary", r.conf.TrafficConf.Name)
}
func getIngressCanaryWeight(ing *netv1.Ingress) int32 {
weightStr, ok := ing.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]
if !ok {
return 0
}
weight, _ := strconv.Atoi(weightStr)
return int32(weight)
}

View File

@ -0,0 +1,60 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package configuration
import (
"context"
"fmt"
"github.com/openkruise/rollouts/pkg/util"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"sigs.k8s.io/controller-runtime/pkg/client"
)
const (
// kruise rollout configmap name
RolloutConfigurationName = "kruise-rollout-configuration"
LuaTrafficRoutingIngressTypePrefix = "lua.traffic.routing.ingress"
)
func GetTrafficRoutingIngressLuaScript(client client.Client, iType string) (string, error) {
data, err := getRolloutConfiguration(client)
if err != nil {
return "", err
} else if len(data) == 0 {
return "", nil
}
value, ok := data[fmt.Sprintf("%s.%s", LuaTrafficRoutingIngressTypePrefix, iType)]
if !ok {
return "", nil
}
return value, nil
}
func getRolloutConfiguration(c client.Client) (map[string]string, error) {
cfg := &corev1.ConfigMap{}
err := c.Get(context.TODO(), client.ObjectKey{Namespace: util.GetRolloutNamespace(), Name: RolloutConfigurationName}, cfg)
if err != nil {
if errors.IsNotFound(err) {
return nil, nil
}
return nil, err
}
return cfg.Data, nil
}

View File

@ -0,0 +1,55 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"io/ioutil"
"os"
"path/filepath"
"k8s.io/klog/v2"
)
// patch -> file.Content,
// for example: lua_configuration/trafficrouting_ingress/nginx.lua -> nginx.lua content
var luaConfigurationList map[string]string
func init() {
luaConfigurationList = map[string]string{}
_ = filepath.Walk("./lua_configuration", func(path string, f os.FileInfo, err error) error {
if err != nil {
klog.Errorf("filepath walk ./lua_configuration failed: %s", err.Error())
return err
}
if f.IsDir() {
return nil
}
var data []byte
data, err = ioutil.ReadFile(filepath.Clean(path))
if err != nil {
klog.Errorf("Read file %s failed: %s", path, err.Error())
return err
}
luaConfigurationList[path] = string(data)
return nil
})
klog.Infof("Init Lua Configuration(%s)", DumpJSON(luaConfigurationList))
}
func GetLuaConfigurationContent(key string) string {
return luaConfigurationList[key]
}

190
pkg/util/luamanager/json.go Normal file
View File

@ -0,0 +1,190 @@
/*
Copyright 2022.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package luamanager
import (
"encoding/json"
"errors"
lua "github.com/yuin/gopher-lua"
)
const JsonLibName = "json"
// OpenJson is the module loader function.
func OpenJson(L *lua.LState) int {
mod := L.RegisterModule(JsonLibName, jsonFuncs)
L.Push(mod)
return 1
}
var jsonFuncs = map[string]lua.LGFunction{
"decode": jsonDecode,
"encode": jsonEncode,
}
func jsonDecode(L *lua.LState) int {
str := L.CheckString(1)
value, err := Decode(L, []byte(str))
if err != nil {
L.Push(lua.LNil)
L.Push(lua.LString(err.Error()))
return 2
}
L.Push(value)
return 1
}
func jsonEncode(L *lua.LState) int {
value := L.CheckAny(1)
data, err := Encode(value)
if err != nil {
L.Push(lua.LNil)
L.Push(lua.LString(err.Error()))
return 2
}
L.Push(lua.LString(string(data)))
return 1
}
var (
errNested = errors.New("cannot encode recursively nested tables to JSON")
errSparseArray = errors.New("cannot encode sparse array")
errInvalidKeys = errors.New("cannot encode mixed or invalid key types")
)
type invalidTypeError lua.LValueType
func (i invalidTypeError) Error() string {
return `cannot encode ` + lua.LValueType(i).String() + ` to JSON`
}
// Encode returns the JSON encoding of value.
func Encode(value lua.LValue) ([]byte, error) {
return json.Marshal(jsonValue{
LValue: value,
visited: make(map[*lua.LTable]bool),
})
}
type jsonValue struct {
lua.LValue
visited map[*lua.LTable]bool
}
func (j jsonValue) MarshalJSON() (data []byte, err error) {
switch converted := j.LValue.(type) {
case lua.LBool:
data, err = json.Marshal(bool(converted))
case lua.LNumber:
data, err = json.Marshal(float64(converted))
case *lua.LNilType:
data = []byte(`null`)
case lua.LString:
data, err = json.Marshal(string(converted))
case *lua.LTable:
if j.visited[converted] {
return nil, errNested
}
j.visited[converted] = true
key, value := converted.Next(lua.LNil)
switch key.Type() {
case lua.LTNil: // empty table
data = []byte(`[]`)
case lua.LTNumber:
arr := make([]jsonValue, 0, converted.Len())
expectedKey := lua.LNumber(1)
for key != lua.LNil {
if key.Type() != lua.LTNumber {
err = errInvalidKeys
return
}
if expectedKey != key {
err = errSparseArray
return
}
arr = append(arr, jsonValue{value, j.visited})
expectedKey++
key, value = converted.Next(key)
}
data, err = json.Marshal(arr)
case lua.LTString:
obj := make(map[string]jsonValue)
for key != lua.LNil {
if key.Type() != lua.LTString {
err = errInvalidKeys
return
}
obj[key.String()] = jsonValue{value, j.visited}
key, value = converted.Next(key)
}
data, err = json.Marshal(obj)
default:
err = errInvalidKeys
}
default:
err = invalidTypeError(j.LValue.Type())
}
return
}
// Decode converts the JSON encoded data to Lua values.
func Decode(L *lua.LState, data []byte) (lua.LValue, error) {
var value interface{}
err := json.Unmarshal(data, &value)
if err != nil {
return nil, err
}
return DecodeValue(L, value), nil
}
// DecodeValue converts the value to a Lua value.
//
// This function only converts values that the encoding/json package decodes to.
// All other values will return lua.LNil.
func DecodeValue(L *lua.LState, value interface{}) lua.LValue {
switch converted := value.(type) {
case bool:
return lua.LBool(converted)
case float64:
return lua.LNumber(converted)
case string:
return lua.LString(converted)
case json.Number:
return lua.LString(converted)
case []interface{}:
arr := L.CreateTable(len(converted), 0)
for _, item := range converted {
arr.Append(DecodeValue(L, item))
}
return arr
case map[string]interface{}:
tbl := L.CreateTable(0, len(converted))
for key, item := range converted {
tbl.RawSetH(lua.LString(key), DecodeValue(L, item))
}
return tbl
case nil:
return lua.LNil
}
return lua.LNil
}

View File

@ -0,0 +1,93 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package luamanager
import (
"context"
"encoding/json"
"time"
lua "github.com/yuin/gopher-lua"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
type LuaManager struct{}
func (m *LuaManager) RunLuaScript(obj *unstructured.Unstructured, script string) (*lua.LState, error) {
l := lua.NewState(lua.Options{SkipOpenLibs: true})
defer l.Close()
for _, pair := range []struct {
n string
f lua.LGFunction
}{
{lua.MathLibName, lua.OpenMath},
{lua.BaseLibName, lua.OpenBase},
{lua.TabLibName, lua.OpenTable},
{lua.StringLibName, lua.OpenString},
{JsonLibName, OpenJson},
} {
if err := l.CallByParam(lua.P{
Fn: l.NewFunction(pair.f),
NRet: 0,
Protect: true,
}, lua.LString(pair.n)); err != nil {
return nil, err
}
}
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
l.SetContext(ctx)
objectValue := decodeValue(l, obj.Object)
l.SetGlobal("obj", objectValue)
err := l.DoString(script)
return l, err
}
func decodeValue(L *lua.LState, value interface{}) lua.LValue {
switch converted := value.(type) {
case bool:
return lua.LBool(converted)
case float64:
return lua.LNumber(converted)
case string:
return lua.LString(converted)
case json.Number:
return lua.LString(converted)
case int:
return lua.LNumber(converted)
case int32:
return lua.LNumber(converted)
case int64:
return lua.LNumber(converted)
case []interface{}:
arr := L.CreateTable(len(converted), 0)
for _, item := range converted {
arr.Append(decodeValue(L, item))
}
return arr
case map[string]interface{}:
tbl := L.CreateTable(0, len(converted))
for key, item := range converted {
tbl.RawSetH(lua.LString(key), decodeValue(L, item))
}
return tbl
case nil:
return lua.LNil
}
return lua.LNil
}

View File

@ -0,0 +1,140 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package luamanager
import (
"encoding/json"
"fmt"
"testing"
rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
"github.com/openkruise/rollouts/pkg/util"
lua "github.com/yuin/gopher-lua"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
luajson "layeh.com/gopher-json"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
)
func TestRunLuaScript(t *testing.T) {
cases := []struct {
name string
getObj func() *unstructured.Unstructured
getLuaScript func() string
expectResult func() string
}{
{
name: "lua script test 1",
getObj: func() *unstructured.Unstructured {
type LuaData struct {
Annotations map[string]string
Weight string
Matches []rolloutv1alpha1.HttpRouteMatch
}
data := &LuaData{
Annotations: map[string]string{
"kubernetes.io/ingress.class": "nginx",
},
Weight: "0",
Matches: []rolloutv1alpha1.HttpRouteMatch{
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "user_id",
Value: "123456",
},
},
},
},
}
unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data)
if err != nil {
fmt.Println("to unstructured failed", err.Error())
}
obj := &unstructured.Unstructured{Object: unObj}
return obj
},
getLuaScript: func() string {
return `
annotations = obj.annotations
annotations["weight"] = obj.weight
if ( not obj.matches )
then
return annotations
end
for _,match in pairs(obj.matches)
do
if ( not (match or match.headers) )
then
return annotations
end
for _,header in pairs(match.headers)
do
if ( not header )
then
return annotations
end
condition = {}
condition["name"] = header.name
condition["value"] = header.value
annotations["condition"] = json.encode(condition)
end
end
return annotations
`
},
expectResult: func() string {
obj := map[string]string{
"weight": "0",
"kubernetes.io/ingress.class": "nginx",
"condition": `{"name":"user_id","value":"123456"}`,
}
return util.DumpJSON(obj)
},
},
}
luaManager := &LuaManager{}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
fmt.Println(cs.name)
l, err := luaManager.RunLuaScript(cs.getObj(), cs.getLuaScript())
if err != nil {
t.Fatalf("RunLuaScript failed: %s", err.Error())
return
}
returnValue := l.Get(-1)
if returnValue.Type() != lua.LTTable {
t.Fatalf("expect table output from Lua script, not %s", returnValue.Type().String())
}
jsonBytes, err := luajson.Encode(returnValue)
if err != nil {
t.Fatalf("encode failed: %s", err.Error())
return
}
newObj := map[string]string{}
err = json.Unmarshal(jsonBytes, &newObj)
if err != nil {
t.Fatalf("Unmarshal failed: %s", err.Error())
return
}
if util.DumpJSON(newObj) != cs.expectResult() {
t.Fatalf("expect(%s), but get (%s)", cs.expectResult(), util.DumpJSON(newObj))
}
})
}
}

26
pkg/util/meta.go Normal file
View File

@ -0,0 +1,26 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import "os"
func GetRolloutNamespace() string {
if ns := os.Getenv("POD_NAMESPACE"); len(ns) > 0 {
return ns
}
return "kruise-rollout"
}

View File

@ -193,9 +193,6 @@ func validateRolloutSpecCanaryTraffic(traffic *appsv1alpha1.TrafficRouting, fldP
if traffic.Ingress.Name == "" { if traffic.Ingress.Name == "" {
errList = append(errList, field.Invalid(fldPath.Child("Ingress"), traffic.Ingress, "TrafficRouting.Ingress.Ingress cannot be empty")) errList = append(errList, field.Invalid(fldPath.Child("Ingress"), traffic.Ingress, "TrafficRouting.Ingress.Ingress cannot be empty"))
} }
if traffic.Ingress.ClassType != "" && traffic.Ingress.ClassType != "nginx" {
errList = append(errList, field.Invalid(fldPath.Child("Ingress"), traffic.Ingress, "TrafficRouting.Ingress.ClassType only support nginx"))
}
} }
if traffic.Gateway != nil { if traffic.Gateway != nil {
if traffic.Gateway.HTTPRouteName == nil || *traffic.Gateway.HTTPRouteName == "" { if traffic.Gateway.HTTPRouteName == nil || *traffic.Gateway.HTTPRouteName == "" {
@ -214,8 +211,8 @@ func validateRolloutSpecCanarySteps(steps []appsv1alpha1.CanaryStep, fldPath *fi
for i := range steps { for i := range steps {
s := &steps[i] s := &steps[i]
if isTraffic && s.Weight == nil { if isTraffic && s.Weight == nil && len(s.Matches) == 0 {
return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `weight cannot be empty for traffic routing`)} return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `weight or matches cannot be empty for traffic routing`)}
} else if s.Weight == nil && s.Replicas == nil { } else if s.Weight == nil && s.Replicas == nil {
return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `weight and replicas cannot be empty at the same time`)} return field.ErrorList{field.Invalid(fldPath.Index(i).Child("steps"), steps, `weight and replicas cannot be empty at the same time`)}
} }
@ -231,7 +228,7 @@ func validateRolloutSpecCanarySteps(steps []appsv1alpha1.CanaryStep, fldPath *fi
for i := 1; i < stepCount; i++ { for i := 1; i < stepCount; i++ {
prev := &steps[i-1] prev := &steps[i-1]
curr := &steps[i] curr := &steps[i]
if isTraffic && *curr.Weight < *prev.Weight { if isTraffic && curr.Weight != nil && prev.Weight != nil && *curr.Weight < *prev.Weight {
return field.ErrorList{field.Invalid(fldPath.Child("Weight"), steps, `Steps.Weight must be a non decreasing sequence`)} return field.ErrorList{field.Invalid(fldPath.Child("Weight"), steps, `Steps.Weight must be a non decreasing sequence`)}
} }

View File

@ -232,15 +232,6 @@ func TestRolloutValidateCreate(t *testing.T) {
// return []client.Object{object} // return []client.Object{object}
// }, // },
//}, //},
{
Name: "Wrong Traffic type",
Succeed: false,
GetObject: func() []client.Object {
object := rollout.DeepCopy()
object.Spec.Strategy.Canary.TrafficRoutings[0].Ingress.ClassType = "Whatever"
return []client.Object{object}
},
},
/**************************************************************** /****************************************************************
The following cases are conflict cases The following cases are conflict cases
***************************************************************/ ***************************************************************/

View File

@ -1639,10 +1639,324 @@ var _ = SIGDescribe("Rollout", func() {
Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred()) Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation) WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation)
}) })
It("V1->V2: A/B testing, header & cookies", func() {
finder := util.NewControllerFinder(k8sClient)
By("Creating Rollout...")
rollout := &rolloutsv1alpha1.Rollout{}
Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred())
headerType := gatewayv1alpha2.HeaderMatchRegularExpression
replica1 := intstr.FromInt(1)
replica2 := intstr.FromInt(2)
rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{
{
Matches: []rolloutsv1alpha1.HttpRouteMatch{
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Type: &headerType,
Name: "user_id",
Value: "123456",
},
},
},
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "canary-by-cookie",
Value: "demo",
},
},
},
},
Pause: rolloutsv1alpha1.RolloutPause{},
Replicas: &replica1,
},
{
Weight: utilpointer.Int32(30),
Replicas: &replica2,
Pause: rolloutsv1alpha1.RolloutPause{},
},
}
CreateObject(rollout)
By("Creating workload and waiting for all pods ready...")
// service
service := &v1.Service{}
Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred())
CreateObject(service)
// ingress
ingress := &netv1.Ingress{}
Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred())
CreateObject(ingress)
// workload
workload := &apps.Deployment{}
Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred())
workload.Spec.Replicas = utilpointer.Int32(3)
CreateObject(workload)
WaitDeploymentAllPodsReady(workload)
rss, err := finder.GetReplicaSetsForDeployment(workload)
Expect(err).NotTo(HaveOccurred())
Expect(len(rss)).Should(BeNumerically("==", 1))
stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey]
// v1 -> v2, start rollout action
newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"})
workload.Spec.Template.Spec.Containers[0].Env = newEnvs
UpdateDeployment(workload)
By("Update deployment image from(version1) -> to(version2)")
time.Sleep(time.Second * 3)
// wait step 1 complete
WaitRolloutCanaryStepPaused(rollout.Name, 1)
// canary deployment
cWorkload, err := GetCanaryDeployment(workload)
Expect(err).NotTo(HaveOccurred())
crss, err := finder.GetReplicaSetsForDeployment(cWorkload)
Expect(err).NotTo(HaveOccurred())
Expect(len(crss)).Should(BeNumerically("==", 1))
canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey]
// check rollout status
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing))
Expect(rollout.Status.StableRevision).Should(Equal(stableRevision))
Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision))
Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1))
// check stable, canary service & ingress
// stable service
Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision))
//canary service
cService := &v1.Service{}
Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred())
Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision))
// canary ingress
cIngress := &netv1.Ingress{}
Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("user_id"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header-pattern", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("123456"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-cookie", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("demo"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(""))
// canary deployment
cWorkload, err = GetCanaryDeployment(workload)
Expect(err).NotTo(HaveOccurred())
Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 1))
Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 1))
// resume rollout canary
ResumeRolloutCanary(rollout.Name)
// wait step 2 complete
WaitRolloutCanaryStepPaused(rollout.Name, 2)
// canary ingress
cIngress = &netv1.Ingress{}
Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(""))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header-pattern", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(""))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-cookie", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(""))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("30"))
// canary deployment
cWorkload, err = GetCanaryDeployment(workload)
Expect(err).NotTo(HaveOccurred())
Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 2))
Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 2))
// resume rollout canary
ResumeRolloutCanary(rollout.Name)
// wait rollout complete
WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy)
klog.Infof("rollout(%s) completed, and check", namespace)
// check service & ingress & deployment
// ingress
Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred())
cIngress = &netv1.Ingress{}
Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred())
// service
Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(""))
cService = &v1.Service{}
Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred())
// deployment
Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
Expect(workload.Spec.Paused).Should(BeFalse())
Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas))
Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
for _, env := range workload.Spec.Template.Spec.Containers[0].Env {
if env.Name == "NODE_NAME" {
Expect(env.Value).Should(Equal("version2"))
}
}
// check progressing succeed
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing)
Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded))
Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue)))
Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation)
})
It("V1->V2: A/B testing, aliyun-alb, header & cookies", func() {
finder := util.NewControllerFinder(k8sClient)
configmap := &v1.ConfigMap{}
Expect(ReadYamlToObject("./test_data/rollout/rollout-configuration.yaml", configmap)).ToNot(HaveOccurred())
Expect(k8sClient.Create(context.TODO(), configmap)).NotTo(HaveOccurred())
defer k8sClient.Delete(context.TODO(), configmap)
By("Creating Rollout...")
rollout := &rolloutsv1alpha1.Rollout{}
Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred())
replica1 := intstr.FromInt(1)
replica2 := intstr.FromInt(2)
rollout.Spec.Strategy.Canary.Steps = []rolloutsv1alpha1.CanaryStep{
{
Matches: []rolloutsv1alpha1.HttpRouteMatch{
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "Cookie",
Value: "demo1=value1;demo2=value2",
},
{
Name: "SourceIp",
Value: "192.168.0.0/16;172.16.0.0/16",
},
{
Name: "headername",
Value: "headervalue1;headervalue2",
},
},
},
},
Pause: rolloutsv1alpha1.RolloutPause{},
Replicas: &replica1,
},
{
Weight: utilpointer.Int32(30),
Replicas: &replica2,
Pause: rolloutsv1alpha1.RolloutPause{},
},
}
rollout.Spec.Strategy.Canary.TrafficRoutings[0].Ingress.ClassType = "aliyun-alb"
CreateObject(rollout)
By("Creating workload and waiting for all pods ready...")
// service
service := &v1.Service{}
Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred())
CreateObject(service)
// ingress
ingress := &netv1.Ingress{}
Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred())
ingress.Annotations = map[string]string{}
ingress.Spec.IngressClassName = utilpointer.String("alb")
CreateObject(ingress)
// workload
workload := &apps.Deployment{}
Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred())
workload.Spec.Replicas = utilpointer.Int32(3)
CreateObject(workload)
WaitDeploymentAllPodsReady(workload)
rss, err := finder.GetReplicaSetsForDeployment(workload)
Expect(err).NotTo(HaveOccurred())
Expect(len(rss)).Should(BeNumerically("==", 1))
stableRevision := rss[0].Labels[apps.DefaultDeploymentUniqueLabelKey]
// v1 -> v2, start rollout action
newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"})
workload.Spec.Template.Spec.Containers[0].Env = newEnvs
UpdateDeployment(workload)
By("Update deployment image from(version1) -> to(version2)")
time.Sleep(time.Second * 3)
// wait step 1 complete
WaitRolloutCanaryStepPaused(rollout.Name, 1)
// canary deployment
cWorkload, err := GetCanaryDeployment(workload)
Expect(err).NotTo(HaveOccurred())
crss, err := finder.GetReplicaSetsForDeployment(cWorkload)
Expect(err).NotTo(HaveOccurred())
Expect(len(crss)).Should(BeNumerically("==", 1))
canaryRevision := crss[0].Labels[apps.DefaultDeploymentUniqueLabelKey]
// check rollout status
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
Expect(rollout.Status.Phase).Should(Equal(rolloutsv1alpha1.RolloutPhaseProgressing))
Expect(rollout.Status.StableRevision).Should(Equal(stableRevision))
Expect(rollout.Status.CanaryStatus.PodTemplateHash).Should(Equal(canaryRevision))
Expect(rollout.Status.CanaryStatus.CurrentStepIndex).Should(BeNumerically("==", 1))
// check stable, canary service & ingress
// stable service
Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(stableRevision))
//canary service
cService := &v1.Service{}
Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cService)).NotTo(HaveOccurred())
Expect(cService.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(canaryRevision))
// canary ingress
cIngress := &netv1.Ingress{}
labIngressAnnotationDefaultPrefix := "alb.ingress.kubernetes.io"
Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", labIngressAnnotationDefaultPrefix)]).Should(Equal("true"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", labIngressAnnotationDefaultPrefix)]).Should(Equal(""))
Expect(cIngress.Annotations[fmt.Sprintf("%s/conditions.echoserver-canary", labIngressAnnotationDefaultPrefix)]).Should(Equal(`[{"cookieConfig":{"values":[{"key":"demo1","value":"value1"},{"key":"demo2","value":"value2"}]},"type":"Cookie"},{"sourceIpConfig":{"values":["192.168.0.0/16","172.16.0.0/16"]},"type":"SourceIp"},{"headerConfig":{"key":"headername","values":["headervalue1","headervalue2"]},"type":"Header"}]`))
// canary deployment
cWorkload, err = GetCanaryDeployment(workload)
Expect(err).NotTo(HaveOccurred())
Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 1))
Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 1))
// resume rollout canary
ResumeRolloutCanary(rollout.Name)
// wait step 2 complete
WaitRolloutCanaryStepPaused(rollout.Name, 2)
// canary ingress
cIngress = &netv1.Ingress{}
Expect(GetObject(rollout.Status.CanaryStatus.CanaryService, cIngress)).NotTo(HaveOccurred())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", labIngressAnnotationDefaultPrefix)]).Should(Equal("true"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/conditions.echoserver-canary", labIngressAnnotationDefaultPrefix)]).Should(Equal(""))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", labIngressAnnotationDefaultPrefix)]).Should(Equal("30"))
// canary deployment
cWorkload, err = GetCanaryDeployment(workload)
Expect(err).NotTo(HaveOccurred())
Expect(*cWorkload.Spec.Replicas).Should(BeNumerically("==", 2))
Expect(cWorkload.Status.ReadyReplicas).Should(BeNumerically("==", 2))
// resume rollout canary
ResumeRolloutCanary(rollout.Name)
// wait rollout complete
WaitRolloutStatusPhase(rollout.Name, rolloutsv1alpha1.RolloutPhaseHealthy)
klog.Infof("rollout(%s) completed, and check", namespace)
// check service & ingress & deployment
// ingress
Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred())
cIngress = &netv1.Ingress{}
Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred())
// service
Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(""))
cService = &v1.Service{}
Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred())
// deployment
Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
Expect(workload.Spec.Paused).Should(BeFalse())
Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas))
Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
for _, env := range workload.Spec.Template.Spec.Containers[0].Env {
if env.Name == "NODE_NAME" {
Expect(env.Value).Should(Equal("version2"))
}
}
// check progressing succeed
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
cond := util.GetRolloutCondition(rollout.Status, rolloutsv1alpha1.RolloutConditionProgressing)
Expect(cond.Reason).Should(Equal(rolloutsv1alpha1.ProgressingReasonSucceeded))
Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue)))
Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation)
})
}) })
KruiseDescribe("Canary rollout with Gateway API", func() { KruiseDescribe("Canary rollout with Gateway API", func() {
It("V1->V2: Percentage 20%,40%,60%,80%,90%, and replicas=3", func() { It("V1->V2: Percentage 20%,40%,60%,80%,90%, and replicas=3, Gateway api", func() {
By("Creating Rollout...") By("Creating Rollout...")
rollout := &rolloutsv1alpha1.Rollout{} rollout := &rolloutsv1alpha1.Rollout{}
Expect(ReadYamlToObject("./test_data/gateway/rollout-test.yaml", rollout)).ToNot(HaveOccurred()) Expect(ReadYamlToObject("./test_data/gateway/rollout-test.yaml", rollout)).ToNot(HaveOccurred())
@ -2055,7 +2369,7 @@ var _ = SIGDescribe("Rollout", func() {
} }
}) })
It("V1->V2: Percentage, 20%,40% and continuous release v3", func() { It("V1->V2: cloneSet, Percentage, 20%,40% and continuous release v3", func() {
By("Creating Rollout...") By("Creating Rollout...")
rollout := &rolloutsv1alpha1.Rollout{} rollout := &rolloutsv1alpha1.Rollout{}
Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred()) Expect(ReadYamlToObject("./test_data/rollout/rollout_canary_base.yaml", rollout)).ToNot(HaveOccurred())

View File

@ -0,0 +1,77 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: kruise-rollout-configuration
namespace: kruise-rollout
data:
"lua.traffic.routing.ingress.aliyun-alb": |
function split(input, delimiter)
local arr = {}
string.gsub(input, '[^' .. delimiter ..']+', function(w) table.insert(arr, w) end)
return arr
end
annotations = {}
if ( obj.annotations )
then
annotations = obj.annotations
end
annotations["alb.ingress.kubernetes.io/canary"] = "true"
annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = nil
annotations["alb.ingress.kubernetes.io/canary-weight"] = nil
conditionKey = string.format("alb.ingress.kubernetes.io/conditions.%s", obj.canaryService)
annotations[conditionKey] = nil
if ( obj.weight ~= "-1" )
then
annotations["alb.ingress.kubernetes.io/canary-weight"] = obj.weight
end
if ( not obj.matches )
then
return annotations
end
conditions = {}
match = obj.matches[1]
for _,header in ipairs(match.headers) do
local condition = {}
if ( header.name == "Cookie" )
then
condition.type = "Cookie"
condition.cookieConfig = {}
cookies = split(header.value, ";")
values = {}
for _,cookieStr in ipairs(cookies) do
cookie = split(cookieStr, "=")
value = {}
value.key = cookie[1]
value.value = cookie[2]
table.insert(values, value)
end
condition.cookieConfig.values = values
elseif ( header.name == "SourceIp" )
then
condition.type = "SourceIp"
condition.sourceIpConfig = {}
ips = split(header.value, ";")
values = {}
for _,ip in ipairs(ips) do
table.insert(values, ip)
end
condition.sourceIpConfig.values = values
else
condition.type = "Header"
condition.headerConfig = {}
condition.headerConfig.key = header.name
vals = split(header.value, ";")
values = {}
for _,val in ipairs(vals) do
table.insert(values, val)
end
condition.headerConfig.values = values
end
table.insert(conditions, condition)
end
annotations[conditionKey] = json.encode(conditions)
return annotations