rollouts/pkg/trafficrouting/network/ingress/ingress_test.go

607 lines
20 KiB
Go

/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package ingress
import (
"context"
"fmt"
"reflect"
"testing"
rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
"github.com/openkruise/rollouts/pkg/util"
"github.com/openkruise/rollouts/pkg/util/configuration"
corev1 "k8s.io/api/core/v1"
netv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
utilpointer "k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
)
var (
scheme *runtime.Scheme
demoConf = corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: configuration.RolloutConfigurationName,
Namespace: util.GetRolloutNamespace(),
},
Data: map[string]string{
fmt.Sprintf("%s.nginx", configuration.LuaTrafficRoutingIngressTypePrefix): `
annotations = obj.annotations
annotations["nginx.ingress.kubernetes.io/canary"] = "true"
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil
annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil
if ( obj.weight ~= "-1" )
then
annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight
end
if ( not obj.matches )
then
return annotations
end
for _,match in ipairs(obj.matches) do
header = match.headers[1]
if ( header.name == "canary-by-cookie" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
if ( header.type == "RegularExpression" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
end
end
end
return annotations
`,
fmt.Sprintf("%s.aliyun-alb", configuration.LuaTrafficRoutingIngressTypePrefix): `
function split(input, delimiter)
local arr = {}
string.gsub(input, '[^' .. delimiter ..']+', function(w) table.insert(arr, w) end)
return arr
end
annotations = obj.annotations
annotations["alb.ingress.kubernetes.io/canary"] = "true"
annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = nil
annotations["alb.ingress.kubernetes.io/canary-weight"] = nil
conditionKey = string.format("alb.ingress.kubernetes.io/conditions.%s", obj.canaryService)
annotations[conditionKey] = nil
if ( obj.weight ~= "-1" )
then
annotations["alb.ingress.kubernetes.io/canary-weight"] = obj.weight
end
if ( not obj.matches )
then
return annotations
end
if ( annotations["alb.ingress.kubernetes.io/backend-svcs-protocols"] )
then
protocolobj = json.decode(annotations["alb.ingress.kubernetes.io/backend-svcs-protocols"])
newprotocolobj = {}
for _, v in pairs(protocolobj) do
newprotocolobj[obj.canaryService] = v
end
annotations["alb.ingress.kubernetes.io/backend-svcs-protocols"] = json.encode(newprotocolobj)
end
conditions = {}
match = obj.matches[1]
for _,header in ipairs(match.headers) do
condition = {}
if ( header.name == "Cookie" )
then
condition.type = "Cookie"
condition.cookieConfig = {}
cookies = split(header.value, ";")
values = {}
for _,cookieStr in ipairs(cookies) do
cookie = split(cookieStr, "=")
value = {}
value.key = cookie[1]
value.value = cookie[2]
table.insert(values, value)
end
condition.cookieConfig.values = values
elseif ( header.name == "SourceIp" )
then
condition.type = "SourceIp"
condition.sourceIpConfig = {}
ips = split(header.value, ";")
values = {}
for _,ip in ipairs(ips) do
table.insert(values, ip)
end
condition.sourceIpConfig.values = values
else
condition.type = "Header"
condition.headerConfig = {}
condition.headerConfig.key = header.name
vals = split(header.value, ";")
values = {}
for _,val in ipairs(vals) do
table.insert(values, val)
end
condition.headerConfig.values = values
end
table.insert(conditions, condition)
end
annotations[conditionKey] = json.encode(conditions)
return annotations
`,
},
}
demoIngress = netv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "echoserver",
Annotations: map[string]string{
"kubernetes.io/ingress.class": "nginx",
},
},
Spec: netv1.IngressSpec{
IngressClassName: utilpointer.String("nginx"),
TLS: []netv1.IngressTLS{
{
Hosts: []string{"echoserver.example.com"},
SecretName: "echoserver-name",
},
{
Hosts: []string{"log.example.com"},
SecretName: "log-name",
},
},
Rules: []netv1.IngressRule{
{
Host: "echoserver.example.com",
IngressRuleValue: netv1.IngressRuleValue{
HTTP: &netv1.HTTPIngressRuleValue{
Paths: []netv1.HTTPIngressPath{
{
Path: "/apis/echo",
Backend: netv1.IngressBackend{
Service: &netv1.IngressServiceBackend{
Name: "echoserver",
Port: netv1.ServiceBackendPort{
Number: 80,
},
},
},
},
{
Path: "/apis/other",
Backend: netv1.IngressBackend{
Service: &netv1.IngressServiceBackend{
Name: "other",
Port: netv1.ServiceBackendPort{
Number: 80,
},
},
},
},
},
},
},
},
{
Host: "log.example.com",
IngressRuleValue: netv1.IngressRuleValue{
HTTP: &netv1.HTTPIngressRuleValue{
Paths: []netv1.HTTPIngressPath{
{
Path: "/apis/logs",
Backend: netv1.IngressBackend{
Service: &netv1.IngressServiceBackend{
Name: "echoserver",
Port: netv1.ServiceBackendPort{
Number: 8899,
},
},
},
},
},
},
},
},
},
},
}
)
func init() {
scheme = runtime.NewScheme()
_ = clientgoscheme.AddToScheme(scheme)
_ = rolloutsv1alpha1.AddToScheme(scheme)
}
func TestInitialize(t *testing.T) {
cases := []struct {
name string
getConfigmap func() *corev1.ConfigMap
getIngress func() []*netv1.Ingress
expectIngress func() *netv1.Ingress
}{
{
name: "init test1",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
return []*netv1.Ingress{demoIngress.DeepCopy()}
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0"
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
}
config := Config{
RolloutName: "rollout-demo",
StableService: "echoserver",
CanaryService: "echoserver-canary",
TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{
Name: "echoserver",
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build()
fakeCli.Create(context.TODO(), cs.getConfigmap())
for _, ingress := range cs.getIngress() {
fakeCli.Create(context.TODO(), ingress)
}
controller, err := NewIngressTrafficRouting(fakeCli, config)
if err != nil {
t.Fatalf("NewIngressTrafficRouting failed: %s", err.Error())
return
}
err = controller.Initialize(context.TODO())
if err != nil {
t.Fatalf("Initialize failed: %s", err.Error())
return
}
canaryIngress := &netv1.Ingress{}
err = fakeCli.Get(context.TODO(), client.ObjectKey{Name: "echoserver-canary"}, canaryIngress)
if err != nil {
t.Fatalf("Get canary ingress failed: %s", err.Error())
return
}
expect := cs.expectIngress()
if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) ||
!reflect.DeepEqual(canaryIngress.Spec, expect.Spec) {
t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress))
}
})
}
}
func TestEnsureRoutes(t *testing.T) {
cases := []struct {
name string
getConfigmap func() *corev1.ConfigMap
getIngress func() []*netv1.Ingress
getRoutes func() (*int32, []rolloutsv1alpha1.HttpRouteMatch)
expectIngress func() *netv1.Ingress
ingressType string
}{
{
name: "ensure routes test1",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0"
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
return nil, []rolloutsv1alpha1.HttpRouteMatch{
// header
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "user_id",
Value: "123456",
},
},
},
// cookies
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "canary-by-cookie",
Value: "demo",
},
},
},
}
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo"
expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id"
expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456"
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
{
name: "ensure routes test2",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456"
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
return utilpointer.Int32(40), nil
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "40"
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
{
name: "ensure routes test3",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = "demo"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id"
canary.Annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = "123456"
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
iType := gatewayv1alpha2.HeaderMatchRegularExpression
return nil, []rolloutsv1alpha1.HttpRouteMatch{
// header
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "user_id",
Value: "123*",
Type: &iType,
},
},
},
}
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header"] = "user_id"
expect.Annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = "123*"
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
{
name: "ensure routes test4",
ingressType: "aliyun-alb",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["alb.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["alb.ingress.kubernetes.io/canary-weight"] = "0"
canary.Annotations["alb.ingress.kubernetes.io/backend-svcs-protocols"] = `{"echoserver":"http"}`
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
getRoutes: func() (*int32, []rolloutsv1alpha1.HttpRouteMatch) {
return nil, []rolloutsv1alpha1.HttpRouteMatch{
// header
{
Headers: []gatewayv1alpha2.HTTPHeaderMatch{
{
Name: "Cookie",
Value: "demo1=value1;demo2=value2",
},
{
Name: "SourceIp",
Value: "192.168.0.0/16;172.16.0.0/16",
},
{
Name: "headername",
Value: "headervalue1;headervalue2",
},
},
},
}
},
expectIngress: func() *netv1.Ingress {
expect := demoIngress.DeepCopy()
expect.Name = "echoserver-canary"
expect.Annotations["alb.ingress.kubernetes.io/canary"] = "true"
expect.Annotations["alb.ingress.kubernetes.io/backend-svcs-protocols"] = `{"echoserver-canary":"http"}`
expect.Annotations["alb.ingress.kubernetes.io/conditions.echoserver-canary"] = `[{"cookieConfig":{"values":[{"key":"demo1","value":"value1"},{"key":"demo2","value":"value2"}]},"type":"Cookie"},{"sourceIpConfig":{"values":["192.168.0.0/16","172.16.0.0/16"]},"type":"SourceIp"},{"headerConfig":{"key":"headername","values":["headervalue1","headervalue2"]},"type":"Header"}]`
expect.Spec.Rules[0].HTTP.Paths = expect.Spec.Rules[0].HTTP.Paths[:1]
expect.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
expect.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return expect
},
},
}
config := Config{
RolloutName: "rollout-demo",
StableService: "echoserver",
CanaryService: "echoserver-canary",
TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{
Name: "echoserver",
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build()
fakeCli.Create(context.TODO(), cs.getConfigmap())
for _, ingress := range cs.getIngress() {
fakeCli.Create(context.TODO(), ingress)
}
config.TrafficConf.ClassType = cs.ingressType
controller, err := NewIngressTrafficRouting(fakeCli, config)
if err != nil {
t.Fatalf("NewIngressTrafficRouting failed: %s", err.Error())
return
}
weight, matches := cs.getRoutes()
_, err = controller.EnsureRoutes(context.TODO(), weight, matches)
if err != nil {
t.Fatalf("EnsureRoutes failed: %s", err.Error())
return
}
canaryIngress := &netv1.Ingress{}
err = fakeCli.Get(context.TODO(), client.ObjectKey{Name: "echoserver-canary"}, canaryIngress)
if err != nil {
t.Fatalf("Get canary ingress failed: %s", err.Error())
return
}
expect := cs.expectIngress()
if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) ||
!reflect.DeepEqual(canaryIngress.Spec, expect.Spec) {
t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress))
}
})
}
}
func TestFinalise(t *testing.T) {
cases := []struct {
name string
getConfigmap func() *corev1.ConfigMap
getIngress func() []*netv1.Ingress
expectIngress func() *netv1.Ingress
}{
{
name: "finalise test1",
getConfigmap: func() *corev1.ConfigMap {
return demoConf.DeepCopy()
},
getIngress: func() []*netv1.Ingress {
canary := demoIngress.DeepCopy()
canary.Name = "echoserver-canary"
canary.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
canary.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = "0"
canary.Spec.Rules[0].HTTP.Paths = canary.Spec.Rules[0].HTTP.Paths[:1]
canary.Spec.Rules[0].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
canary.Spec.Rules[1].HTTP.Paths[0].Backend.Service.Name = "echoserver-canary"
return []*netv1.Ingress{demoIngress.DeepCopy(), canary}
},
expectIngress: func() *netv1.Ingress {
return nil
},
},
}
config := Config{
RolloutName: "rollout-demo",
StableService: "echoserver",
CanaryService: "echoserver-canary",
TrafficConf: &rolloutsv1alpha1.IngressTrafficRouting{
Name: "echoserver",
},
}
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build()
fakeCli.Create(context.TODO(), cs.getConfigmap())
for _, ingress := range cs.getIngress() {
fakeCli.Create(context.TODO(), ingress)
}
controller, err := NewIngressTrafficRouting(fakeCli, config)
if err != nil {
t.Fatalf("NewIngressTrafficRouting failed: %s", err.Error())
return
}
err = controller.Finalise(context.TODO())
if err != nil {
t.Fatalf("EnsureRoutes failed: %s", err.Error())
return
}
canaryIngress := &netv1.Ingress{}
err = fakeCli.Get(context.TODO(), client.ObjectKey{Name: "echoserver-canary"}, canaryIngress)
if err != nil {
if cs.expectIngress() == nil && errors.IsNotFound(err) {
return
}
t.Fatalf("Get canary ingress failed: %s", err.Error())
return
}
expect := cs.expectIngress()
if !reflect.DeepEqual(canaryIngress.Annotations, expect.Annotations) ||
!reflect.DeepEqual(canaryIngress.Spec, expect.Spec) {
t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect), util.DumpJSON(canaryIngress))
}
})
}
}