openkruise
/
rollouts
mirror of https://github.com/openkruise/rollouts.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

custom trafficRouting 

Signed-off-by: Kuromesi <blackfacepan@163.com>
This commit is contained in:
Kuromesi 2023-07-12 17:46:35 +08:00
parent e6b31de343
commit e9d3196e32
12 changed files with 381 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/trafficrouting/manager.go
View File

@ -185,7 +185,7 @@ func (m *Manager) DoTrafficRouting(c *TrafficRoutingContext) (bool, error) {
key := fmt.Sprintf("%s.%s", c.Key, trafficRouting.Service)
trController, ok := controllerMap[key].(network.NetworkProvider)
if !ok {
// in case the rollout controller restart accidentally, create a new trafficRouting controller
// in case the rollout controller restart unexpectedly, create a new trafficRouting controller
err := m.InitializeTrafficRouting(c)
if err != nil {
return false, err

44
pkg/trafficrouting/network/custom/custom.go
View File

@ -46,17 +46,6 @@ const (
LuaConfigMap = "kruise-rollout-configuration"
)
type NetworkTrafficRouting struct {
// API Version of the referent
APIVersion string `json:"apiVersion"`
// Kind of the referent
Kind string `json:"kind"`
// Name of the referent
Name string `json:"name"`
// Name of the lua script
Lua string `json:"lua"`
}
type customController struct {
client.Client
conf Config
@ -92,15 +81,26 @@ func (r *customController) Initialize(ctx context.Context) error {
if err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil {
return err
}
// check if lua script exists
script := r.getLuascript(ctx, ref)
if script == "" {
klog.Errorf("failed to get lua script for %s", ref.Kind)
return nil
if _, ok := r.luaScript[ref.Kind]; !ok {
script := r.getLuascript(ctx, ref)
if script == "" {
klog.Errorf("failed to get lua script for %s", ref.Kind)
return nil
}
// is it necessary to consider same kind but different apiversion?
r.luaScript[ref.Kind] = script
}
annotations := obj.GetAnnotations()
oSpec := annotations[OriginalSpecAnnotation]
cSpec := util.DumpJSON(obj.Object["spec"])
if oSpec == cSpec {
continue
}
if err := r.storeObject(obj); err != nil {
klog.Errorf("failed to store object: %s/%s", ref.Kind, ref.Name)
return err
}
// is it necessary to consider same kind but different apiversion?
r.luaScript[ref.Kind] = script
}
return nil
}
@ -116,14 +116,6 @@ func (r *customController) EnsureRoutes(ctx context.Context, strategy *rolloutv1
return false, err
}
specStr := obj.GetAnnotations()[OriginalSpecAnnotation]
if specStr == "" {
err = r.storeObject(obj)
if err != nil {
klog.Errorf("failed to store object: %s/%s", ref.Kind, ref.Name)
return false, err
}
specStr = obj.GetAnnotations()[OriginalSpecAnnotation]
}
var oSpec interface{}
_ = json.Unmarshal([]byte(specStr), &oSpec)
nSpec, err := r.executeLuaForCanary(oSpec, strategy, r.luaScript[ref.Kind])

108
pkg/trafficrouting/network/custom/custom_test.go
View File

@ -23,6 +23,7 @@ import (
"reflect"
"testing"
"github.com/openkruise/rollouts/api/v1alpha1"
rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
"github.com/openkruise/rollouts/pkg/util"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@ -154,12 +155,11 @@ func TestInitialize(t *testing.T) {
return Config{
StableService: "echoserver",
CanaryService: "echoserver-canary",
TrafficConf: []NetworkTrafficRouting{
TrafficConf: []v1alpha1.NetworkRef{
{
APIVersion: "networking.istio.io/v1alpha3",
Kind: "VirtualService",
Name: "echoserver",
Lua: "lua-demo",
},
},
}
@ -185,7 +185,7 @@ func TestInitialize(t *testing.T) {
klog.Errorf(err.Error())
return
}
c, _ := NewCustomController(fakeCli, cs.getConfig(), cs.getLua())
c, _ := NewCustomController(fakeCli, cs.getConfig())
err = c.Initialize(context.TODO())
if err != nil {
t.Fatalf("Initialize failed: %s", err.Error())
@ -250,7 +250,7 @@ func TestEnsureRoutes(t *testing.T) {
"virtual": "test",
}
u.SetAnnotations(annotations)
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"95"},{"destination":{"host":"echoserver-canary","weight":"5"}}]}]}`
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}}]}]}`
var spec interface{}
_ = json.Unmarshal([]byte(specStr), &spec)
u.Object["spec"] = spec
@ -278,7 +278,7 @@ func TestEnsureRoutes(t *testing.T) {
"virtual": "test",
}
u.SetAnnotations(annotations)
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"95"},{"destination":{"host":"echoserver-canary","weight":"5"}}]}]}`
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}]}]}`
var spec interface{}
_ = json.Unmarshal([]byte(specStr), &spec)
u.Object["spec"] = spec
@ -292,66 +292,65 @@ func TestEnsureRoutes(t *testing.T) {
"virtual": "test",
}
u.SetAnnotations(annotations)
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"95"},{"destination":{"host":"echoserver-canary","weight":"5"}}]}]}`
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}]}]}`
var spec interface{}
_ = json.Unmarshal([]byte(specStr), &spec)
u.Object["spec"] = spec
return true, u
},
},
{
name: "test3",
getLua: func() map[string]string {
luaMap := map[string]string{
"lua-demo": luaDemo,
}
return luaMap
},
getRoutes: func() *rolloutsv1alpha1.TrafficRoutingStrategy {
return &rolloutsv1alpha1.TrafficRoutingStrategy{
Weight: utilpointer.Int32(0),
}
},
getUnstructured: func() *unstructured.Unstructured {
u := &unstructured.Unstructured{}
_ = u.UnmarshalJSON([]byte(networkDemo))
annotations := map[string]string{
OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`,
"virtual": "test",
}
u.SetAnnotations(annotations)
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"95"},{"destination":{"host":"echoserver-canary","weight":"5"}}]}]}`
var spec interface{}
_ = json.Unmarshal([]byte(specStr), &spec)
u.Object["spec"] = spec
return u
},
expectInfo: func() (bool, *unstructured.Unstructured) {
u := &unstructured.Unstructured{}
_ = u.UnmarshalJSON([]byte(networkDemo))
annotations := map[string]string{
OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`,
"virtual": "test",
}
u.SetAnnotations(annotations)
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"100"},{"destination":{"host":"echoserver-canary","weight":"0"}}]}]}`
var spec interface{}
_ = json.Unmarshal([]byte(specStr), &spec)
u.Object["spec"] = spec
return false, u
},
},
// {
// name: "test3",
// getLua: func() map[string]string {
// luaMap := map[string]string{
// "lua-demo": luaDemo,
// }
// return luaMap
// },
// getRoutes: func() *rolloutsv1alpha1.TrafficRoutingStrategy {
// return &rolloutsv1alpha1.TrafficRoutingStrategy{
// Weight: utilpointer.Int32(0),
// }
// },
// getUnstructured: func() *unstructured.Unstructured {
// u := &unstructured.Unstructured{}
// _ = u.UnmarshalJSON([]byte(networkDemo))
// annotations := map[string]string{
// OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`,
// "virtual": "test",
// }
// u.SetAnnotations(annotations)
// specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}]}]}`
// var spec interface{}
// _ = json.Unmarshal([]byte(specStr), &spec)
// u.Object["spec"] = spec
// return u
// },
// expectInfo: func() (bool, *unstructured.Unstructured) {
// u := &unstructured.Unstructured{}
// _ = u.UnmarshalJSON([]byte(networkDemo))
// annotations := map[string]string{
// OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`,
// "virtual": "test",
// }
// u.SetAnnotations(annotations)
// specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":100},{"destination":{"host":"echoserver-canary"},"weight":0}]}]}`
// var spec interface{}
// _ = json.Unmarshal([]byte(specStr), &spec)
// u.Object["spec"] = spec
// return false, u
// },
// },
}
config := Config{
RolloutName: "rollout-demo",
StableService: "echoserver",
CanaryService: "echoserver-canary",
TrafficConf: []NetworkTrafficRouting{
TrafficConf: []v1alpha1.NetworkRef{
{
APIVersion: "networking.istio.io/v1alpha3",
Kind: "VirtualService",
Name: "echoserver",
Lua: "lua-demo",
},
},
}
@ -363,9 +362,10 @@ func TestEnsureRoutes(t *testing.T) {
klog.Errorf(err.Error())
return
}
c, _ := NewCustomController(fakeCli, config, cs.getLua())
c, _ := NewCustomController(fakeCli, config)
strategy := cs.getRoutes()
expect1, expect2 := cs.expectInfo()
c.Initialize(context.TODO())
done, err := c.EnsureRoutes(context.TODO(), strategy)
if err != nil {
t.Fatalf("EnsureRoutes failed: %s", err.Error())
@ -394,7 +394,7 @@ func TestFinalise(t *testing.T) {
"virtual": "test",
}
u.SetAnnotations(annotations)
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"100"},{"destination":{"host":"echoserver-canary","weight":"0"}}]}]}`
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":100},{"destination":{"host":"echoserver-canary"},"weight":0}}]}]}`
var spec interface{}
_ = json.Unmarshal([]byte(specStr), &spec)
u.Object["spec"] = spec
@ -404,7 +404,7 @@ func TestFinalise(t *testing.T) {
return Config{
StableService: "echoserver",
CanaryService: "echoserver-canary",
TrafficConf: []rolloutsv1alpha1.NetworkTrafficRouting{
TrafficConf: []v1alpha1.NetworkRef{
{
APIVersion: "networking.istio.io/v1alpha3",
Kind: "VirtualService",
@ -429,7 +429,7 @@ func TestFinalise(t *testing.T) {
klog.Errorf(err.Error())
return
}
c, _ := NewCustomController(fakeCli, cs.getConfig(), "")
c, _ := NewCustomController(fakeCli, cs.getConfig())
err = c.Finalise(context.TODO())
if err != nil {
t.Fatalf("Initialize failed: %s", err.Error())

25
pkg/trafficrouting/network/custom/lua_configuration/networking.istio.io/VirtualService/testdata/test_case1.yaml vendored Normal file
View File

@ -0,0 +1,25 @@
stableService: details
canaryService: details-canary
stableWeight: 90
canaryWeight: 10
matches:
spec:
hosts:
- details
http:
- route:
- destination:
host: details
subset: v1
nSpec:
hosts:
- details
http:
- route:
- destination:
host: details
subset: v1
weight: 90
- destination:
host: details-canary
weight: 10

34
pkg/trafficrouting/network/custom/lua_configuration/networking.istio.io/VirtualService/testdata/test_case2.yaml vendored Normal file
View File

@ -0,0 +1,34 @@
stableService: details
canaryService: details-canary
stableWeight: 90
canaryWeight: 10
matches:
- headers:
- type: RegularExpression
name: end-user
value: kuromesi
spec:
hosts:
- details
http:
- route:
- destination:
host: details
nSpec:
hosts:
- details
http:
- matches:
- headers:
end-user:
regex: kuromesi
route:
- destination:
host: details
weight: 90
- destination:
host: details-canary
weight: 10
- route:
- destination:
host: details

93
pkg/trafficrouting/network/custom/lua_configuration/networking.istio.io/VirtualService/trafficRouting.lua Normal file
View File

@ -0,0 +1,93 @@
-- obj = {
-- -- matches = {
-- -- {
-- -- headers = {
-- -- {
-- -- name = "xxx",
-- -- value = "xxx",
-- -- type = "RegularExpression"
-- -- }
-- -- }
-- -- }
-- -- },
-- spec = {
-- hosts = {
-- "reviews",
-- },
-- http = {
-- {
-- route = {
-- {
-- destination = {
-- host = "reviews",
-- subset = "c1"
-- }
-- }
-- }
-- }
-- }
-- },
-- stableService = "reviews",
-- canaryService = "canary",
-- stableWeight = 90,
-- canaryWeight = 10
-- }
spec = obj.spec
if (obj.matches) then
for _, match in ipairs(obj.matches) do
local route = {}
route["matches"] = {}
for key, value in pairs(match) do
local vsMatch = {}
vsMatch[key] = {}
for _, rule in ipairs(value) do
if rule["type"] == "RegularExpression"
then
matchType = "regex"
else
matchType = "exact"
end
vsMatch[key][rule["name"]] = {}
vsMatch[key][rule["name"]][matchType] = rule["value"]
end
table.insert(route["matches"], vsMatch)
end
route["route"] = {
{
destination = {
host = obj.stableService,
},
weight = obj.stableWeight,
},
{
destination = {
host = obj.canaryService,
},
weight = obj.canaryWeight,
}
}
table.insert(spec.http, 1, route)
end
return spec
end
for i, rule in ipairs(obj.spec.http) do
for _, route in ipairs(rule.route) do
local destination = route.destination
if destination.host == obj.stableService then
route.weight = obj.stableWeight
-- destination.weight = obj.stableWeight
local canary = {
destination = {
host = obj.canaryService,
},
weight = obj.canaryWeight,
}
table.insert(rule.route, canary)
end
end
end
return spec

36
pkg/trafficrouting/network/custom/lua_configuration/trafficrouting_ingress/aliyun-alb.lua Normal file
View File

@ -0,0 +1,36 @@
annotations = {}
if ( obj.annotations )
then
annotations = obj.annotations
end
annotations["alb.ingress.kubernetes.io/canary"] = "true"
annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = nil
annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = nil
annotations["alb.ingress.kubernetes.io/canary-weight"] = nil
annotations["alb.ingress.kubernetes.io/order"] = "1"
if ( obj.weight ~= "-1" )
then
annotations["alb.ingress.kubernetes.io/canary-weight"] = obj.weight
end
if ( not obj.matches )
then
return annotations
end
for _,match in ipairs(obj.matches) do
local header = match.headers[1]
if ( header.name == "canary-by-cookie" )
then
annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = header.value
else
annotations["alb.ingress.kubernetes.io/canary-by-header"] = header.name
if ( header.type == "RegularExpression" )
then
annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
else
annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = header.value
end
end
end
return annotations

12
lua_configuration/networking.k8s.io/ingress/trafficRouting.lua → pkg/trafficrouting/network/custom/lua_configuration/trafficrouting_ingress/higress.lua
View File

@ -1,9 +1,8 @@
annotations = {}
spec = {}
-- obj.annotations is ingress annotations, it is recommended not to remove the part of the lua script, it must be kept
if ( obj.obj.metadata.annotations )
if ( obj.annotations )
then
annotations = obj.obj.metadata.annotations
annotations = obj.annotations
end
-- indicates the ingress is nginx canary api
annotations["nginx.ingress.kubernetes.io/canary"] = "true"
@ -22,10 +21,7 @@ end
-- if don't contains headers, immediate return annotations
if ( not obj.matches )
then
return {
spec = spec,
annotations = annotations
}
return annotations
end
-- headers & cookie apis
-- traverse matches
@ -47,4 +43,4 @@ for _,match in ipairs(obj.matches) do
end
end
-- must be return annotations
return spec
return annotations

45
pkg/trafficrouting/network/custom/lua_configuration/trafficrouting_ingress/mse.lua Normal file
View File

@ -0,0 +1,45 @@
function split(input, delimiter)
local arr = {}
string.gsub(input, '[^' .. delimiter ..']+', function(w) table.insert(arr, w) end)
return arr
end
annotations = obj.annotations
annotations["nginx.ingress.kubernetes.io/canary"] = "true"
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil
annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil
if ( obj.weight ~= "-1" )
then
annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight
end
if ( obj.requestHeaderModifier )
then
local str = ''
for _,header in ipairs(obj.requestHeaderModifier.set) do
str = str..string.format("%s %s", header.name, header.value)
end
annotations["mse.ingress.kubernetes.io/request-header-control-update"] = str
end
if ( not obj.matches )
then
return annotations
end
for _,match in ipairs(obj.matches) do
header = match.headers[1]
if ( header.name == "canary-by-cookie" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
if ( header.type == "RegularExpression" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
end
end
end
return annotations

46
pkg/trafficrouting/network/custom/lua_configuration/trafficrouting_ingress/nginx.lua Normal file
View File

@ -0,0 +1,46 @@
annotations = {}
-- obj.annotations is ingress annotations, it is recommended not to remove the part of the lua script, it must be kept
if ( obj.annotations )
then
annotations = obj.annotations
end
-- indicates the ingress is nginx canary api
annotations["nginx.ingress.kubernetes.io/canary"] = "true"
-- First, set all canary api to nil
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil
annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil
-- if rollout.spec.strategy.canary.steps.weight is nil, obj.weight will be -1,
-- then we need remove the canary-weight annotation
if ( obj.weight ~= "-1" )
then
annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight
end
-- if don't contains headers, immediate return annotations
if ( not obj.matches )
then
return annotations
end
-- headers & cookie apis
-- traverse matches
for _,match in ipairs(obj.matches) do
local header = match.headers[1]
-- cookie
if ( header.name == "canary-by-cookie" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
-- if regular expression
if ( header.type == "RegularExpression" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
end
end
end
-- must be return annotations
return annotations

2
pkg/util/lua_configuration.go
View File

@ -35,7 +35,7 @@ func init() {
klog.Warningf("filepath walk ./lua_configuration failed: %s", err.Error())
return err
}
if f.IsDir() {
if f.IsDir() || filepath.Ext(path) != ".lua" {
return nil
}
var data []byte

24
scripts/curl.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
URL=$1
TIMES=$2
STRING1="v1"
STRING2="v2"
COUNT1=0
COUNT2=0
for ((i=1; i<=$TIMES; i++))
do
response=$(curl -s "$URL")
if [[ $response == *"$STRING1"* ]]
then
((COUNT1++))
elif [[ $response == *"$STRING2"* ]]
then
((COUNT2++))
fi
done
echo "Total query: $2"
echo "'$STRING1': $COUNT1"
echo "'$STRING2': $COUNT2"