make some improvements
Signed-off-by: Kuromesi <blackfacepan@163.com>
This commit is contained in:
parent
6e57766cf3
commit
4bf65e3a71
|
|
@ -0,0 +1,225 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/openkruise/rollouts/api/v1alpha1"
|
||||||
|
"github.com/openkruise/rollouts/pkg/trafficrouting/network/custom"
|
||||||
|
"github.com/openkruise/rollouts/pkg/util/luamanager"
|
||||||
|
lua "github.com/yuin/gopher-lua"
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime"
|
||||||
|
"sigs.k8s.io/yaml"
|
||||||
|
|
||||||
|
utilpointer "k8s.io/utils/pointer"
|
||||||
|
)
|
||||||
|
|
||||||
|
type TestCase struct {
|
||||||
|
Rollout *v1alpha1.Rollout `json:"rollout,omitempty"`
|
||||||
|
TrafficRouting *v1alpha1.TrafficRouting `json:"trafficRouting,omitempty"`
|
||||||
|
Original *unstructured.Unstructured `json:"original,omitempty"`
|
||||||
|
Expected []*unstructured.Unstructured `json:"expected,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// convert testdata to lua object for debugging
|
||||||
|
func main() {
|
||||||
|
err := PathWalk()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func PathWalk() error {
|
||||||
|
err := filepath.Walk("./", func(path string, f os.FileInfo, err error) error {
|
||||||
|
if !strings.Contains(path, "trafficRouting.lua") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to walk path: %s", err.Error())
|
||||||
|
}
|
||||||
|
dir := filepath.Dir(path)
|
||||||
|
err = filepath.Walk(filepath.Join(dir, "testdata"), func(path string, info os.FileInfo, err error) error {
|
||||||
|
if !info.IsDir() && filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" {
|
||||||
|
fmt.Printf("--- walking path: %s ---\n", path)
|
||||||
|
err = ObjectToTable(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to convert object to table: %s", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to walk path: %s", err.Error())
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to walk path: %s", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// convert a testcase object to lua table for debug
|
||||||
|
func ObjectToTable(path string) error {
|
||||||
|
dir, file := filepath.Split(path)
|
||||||
|
testCase, err := getLuaTestCase(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get lua testcase: %s", err)
|
||||||
|
}
|
||||||
|
uList := make(map[string]interface{})
|
||||||
|
rollout := testCase.Rollout
|
||||||
|
trafficRouting := testCase.TrafficRouting
|
||||||
|
if rollout != nil {
|
||||||
|
steps := rollout.Spec.Strategy.Canary.Steps
|
||||||
|
for i, step := range steps {
|
||||||
|
weight := step.TrafficRoutingStrategy.Weight
|
||||||
|
if step.TrafficRoutingStrategy.Weight == nil {
|
||||||
|
weight = utilpointer.Int32(-1)
|
||||||
|
}
|
||||||
|
var canaryService string
|
||||||
|
stableService := rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service
|
||||||
|
// if rollout.Spec.Strategy.Canary.TrafficRoutings[0].CreateCanaryService {
|
||||||
|
canaryService = fmt.Sprintf("%s-canary", stableService)
|
||||||
|
// } else {
|
||||||
|
// canaryService = stableService
|
||||||
|
// }
|
||||||
|
data := &custom.LuaData{
|
||||||
|
Data: custom.Data{
|
||||||
|
Labels: testCase.Original.GetLabels(),
|
||||||
|
Annotations: testCase.Original.GetAnnotations(),
|
||||||
|
Spec: testCase.Original.Object["spec"],
|
||||||
|
},
|
||||||
|
Matches: step.TrafficRoutingStrategy.Matches,
|
||||||
|
CanaryWeight: *weight,
|
||||||
|
StableWeight: 100 - *weight,
|
||||||
|
CanaryService: canaryService,
|
||||||
|
StableService: stableService,
|
||||||
|
}
|
||||||
|
uList[fmt.Sprintf("step_%d", i)] = data
|
||||||
|
}
|
||||||
|
} else if trafficRouting != nil {
|
||||||
|
weight := trafficRouting.Spec.Strategy.Weight
|
||||||
|
if weight == nil {
|
||||||
|
weight = utilpointer.Int32(-1)
|
||||||
|
}
|
||||||
|
var canaryService string
|
||||||
|
stableService := trafficRouting.Spec.ObjectRef[0].Service
|
||||||
|
canaryService = stableService
|
||||||
|
data := &custom.LuaData{
|
||||||
|
Data: custom.Data{
|
||||||
|
Labels: testCase.Original.GetLabels(),
|
||||||
|
Annotations: testCase.Original.GetAnnotations(),
|
||||||
|
Spec: testCase.Original.Object["spec"],
|
||||||
|
},
|
||||||
|
Matches: trafficRouting.Spec.Strategy.Matches,
|
||||||
|
CanaryWeight: *weight,
|
||||||
|
StableWeight: 100 - *weight,
|
||||||
|
CanaryService: canaryService,
|
||||||
|
StableService: stableService,
|
||||||
|
}
|
||||||
|
uList["steps_0"] = data
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("neither rollout nor trafficRouting defined in test case: %s", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
objStr, err := executeLua(uList)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to execute lua: %s", err.Error())
|
||||||
|
}
|
||||||
|
filePath := fmt.Sprintf("%s%s_obj.lua", dir, strings.Split(file, ".")[0])
|
||||||
|
fileStream, err := os.OpenFile(filePath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open file: %s", err)
|
||||||
|
}
|
||||||
|
defer fileStream.Close()
|
||||||
|
_, err = io.WriteString(fileStream, objStr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to WriteString %s", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getLuaTestCase(path string) (*TestCase, error) {
|
||||||
|
yamlFile, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
luaTestCase := &TestCase{}
|
||||||
|
err = yaml.Unmarshal(yamlFile, luaTestCase)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return luaTestCase, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func executeLua(steps map[string]interface{}) (string, error) {
|
||||||
|
luaManager := &luamanager.LuaManager{}
|
||||||
|
unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&steps)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to convert to unstructured: %s", err)
|
||||||
|
}
|
||||||
|
u := &unstructured.Unstructured{Object: unObj}
|
||||||
|
script := `
|
||||||
|
function serialize(obj, isKey)
|
||||||
|
local lua = ""
|
||||||
|
local t = type(obj)
|
||||||
|
if t == "number" then
|
||||||
|
lua = lua .. obj
|
||||||
|
elseif t == "boolean" then
|
||||||
|
lua = lua .. tostring(obj)
|
||||||
|
elseif t == "string" then
|
||||||
|
if isKey then
|
||||||
|
lua = lua .. string.format("%s", obj)
|
||||||
|
else
|
||||||
|
lua = lua .. string.format("%q", obj)
|
||||||
|
end
|
||||||
|
elseif t == "table" then
|
||||||
|
lua = lua .. "{"
|
||||||
|
for k, v in pairs(obj) do
|
||||||
|
if type(k) == "string" then
|
||||||
|
lua = lua .. serialize(k, true) .. "=" .. serialize(v, false) .. ","
|
||||||
|
else
|
||||||
|
lua = lua .. serialize(v, false) .. ","
|
||||||
|
end
|
||||||
|
end
|
||||||
|
local metatable = getmetatable(obj)
|
||||||
|
if metatable ~= nil and type(metatable.__index) == "table" then
|
||||||
|
for k, v in pairs(metatable.__index) do
|
||||||
|
if type(k) == "string" then
|
||||||
|
lua = lua .. serialize(k, true) .. "=" .. serialize(v, false) .. ","
|
||||||
|
else
|
||||||
|
lua = lua .. serialize(v, false) .. ","
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
lua = lua .. "}"
|
||||||
|
elseif t == "nil" then
|
||||||
|
return nil
|
||||||
|
else
|
||||||
|
error("can not serialize a " .. t .. " type.")
|
||||||
|
end
|
||||||
|
return lua
|
||||||
|
end
|
||||||
|
|
||||||
|
function table2string(tablevalue)
|
||||||
|
local stringtable = "steps=" .. serialize(tablevalue)
|
||||||
|
print(stringtable)
|
||||||
|
return stringtable
|
||||||
|
end
|
||||||
|
return table2string(obj)
|
||||||
|
`
|
||||||
|
l, err := luaManager.RunLuaScript(u, script)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to run lua script: %s", err)
|
||||||
|
}
|
||||||
|
returnValue := l.Get(-1)
|
||||||
|
if returnValue.Type() == lua.LTString {
|
||||||
|
return returnValue.String(), nil
|
||||||
|
} else {
|
||||||
|
return "", fmt.Errorf("unexpected lua output type")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -266,7 +266,7 @@ func newNetworkProvider(c client.Client, con *TrafficRoutingContext, sService, c
|
||||||
trafficRouting := con.ObjectRef[0]
|
trafficRouting := con.ObjectRef[0]
|
||||||
if trafficRouting.CustomNetworkRefs != nil {
|
if trafficRouting.CustomNetworkRefs != nil {
|
||||||
return custom.NewCustomController(c, custom.Config{
|
return custom.NewCustomController(c, custom.Config{
|
||||||
RolloutName: con.Key,
|
Key: con.Key,
|
||||||
RolloutNs: con.Namespace,
|
RolloutNs: con.Namespace,
|
||||||
CanaryService: cService,
|
CanaryService: cService,
|
||||||
StableService: sService,
|
StableService: sService,
|
||||||
|
|
|
||||||
|
|
@ -26,7 +26,6 @@ import (
|
||||||
"k8s.io/apimachinery/pkg/api/errors"
|
"k8s.io/apimachinery/pkg/api/errors"
|
||||||
"k8s.io/klog/v2"
|
"k8s.io/klog/v2"
|
||||||
|
|
||||||
"github.com/openkruise/rollouts/api/v1alpha1"
|
|
||||||
rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
|
rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
|
||||||
"github.com/openkruise/rollouts/pkg/trafficrouting/network"
|
"github.com/openkruise/rollouts/pkg/trafficrouting/network"
|
||||||
"github.com/openkruise/rollouts/pkg/util"
|
"github.com/openkruise/rollouts/pkg/util"
|
||||||
|
|
@ -47,6 +46,14 @@ const (
|
||||||
LuaConfigMap = "kruise-rollout-configuration"
|
LuaConfigMap = "kruise-rollout-configuration"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type LuaData struct {
|
||||||
|
Data Data
|
||||||
|
CanaryWeight int32
|
||||||
|
StableWeight int32
|
||||||
|
Matches []rolloutv1alpha1.HttpRouteMatch
|
||||||
|
CanaryService string
|
||||||
|
StableService string
|
||||||
|
}
|
||||||
type Data struct {
|
type Data struct {
|
||||||
Spec interface{} `json:"spec,omitempty"`
|
Spec interface{} `json:"spec,omitempty"`
|
||||||
Labels map[string]string `json:"labels,omitempty"`
|
Labels map[string]string `json:"labels,omitempty"`
|
||||||
|
|
@ -60,7 +67,7 @@ type customController struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
RolloutName string
|
Key string
|
||||||
RolloutNs string
|
RolloutNs string
|
||||||
CanaryService string
|
CanaryService string
|
||||||
StableService string
|
StableService string
|
||||||
|
|
@ -151,7 +158,6 @@ func (r *customController) Finalise(ctx context.Context) error {
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// when one failed how to proceed?
|
|
||||||
if err := r.restoreObject(obj); err != nil {
|
if err := r.restoreObject(obj); err != nil {
|
||||||
klog.Errorf("failed to restore object: %s/%s", ref.Kind, ref.Name)
|
klog.Errorf("failed to restore object: %s/%s", ref.Kind, ref.Name)
|
||||||
return err
|
return err
|
||||||
|
|
@ -211,33 +217,18 @@ func (r *customController) restoreObject(obj *unstructured.Unstructured) error {
|
||||||
func (r *customController) executeLuaForCanary(spec Data, strategy *rolloutv1alpha1.TrafficRoutingStrategy, luaScript string) (Data, error) {
|
func (r *customController) executeLuaForCanary(spec Data, strategy *rolloutv1alpha1.TrafficRoutingStrategy, luaScript string) (Data, error) {
|
||||||
weight := strategy.Weight
|
weight := strategy.Weight
|
||||||
matches := strategy.Matches
|
matches := strategy.Matches
|
||||||
rollout := &v1alpha1.Rollout{}
|
|
||||||
if err := r.Get(context.TODO(), types.NamespacedName{Namespace: r.conf.RolloutNs, Name: r.conf.RolloutName}, rollout); err != nil {
|
|
||||||
klog.Errorf("failed to get rollout/%s when execute custom network provider lua script", r.conf.RolloutName)
|
|
||||||
return Data{}, err
|
|
||||||
}
|
|
||||||
if weight == nil {
|
if weight == nil {
|
||||||
// the lua script does not have a pointer type,
|
// the lua script does not have a pointer type,
|
||||||
// so we need to pass weight=-1 to indicate the case where weight is nil.
|
// so we need to pass weight=-1 to indicate the case where weight is nil.
|
||||||
weight = utilpointer.Int32(-1)
|
weight = utilpointer.Int32(-1)
|
||||||
}
|
}
|
||||||
type LuaData struct {
|
|
||||||
Data Data
|
|
||||||
CanaryWeight int32
|
|
||||||
StableWeight int32
|
|
||||||
Matches []rolloutv1alpha1.HttpRouteMatch
|
|
||||||
CanaryService string
|
|
||||||
StableService string
|
|
||||||
PatchPodMetadata *rolloutv1alpha1.PatchPodTemplateMetadata
|
|
||||||
}
|
|
||||||
data := &LuaData{
|
data := &LuaData{
|
||||||
Data: spec,
|
Data: spec,
|
||||||
CanaryWeight: *weight,
|
CanaryWeight: *weight,
|
||||||
StableWeight: 100 - *weight,
|
StableWeight: 100 - *weight,
|
||||||
Matches: matches,
|
Matches: matches,
|
||||||
CanaryService: r.conf.CanaryService,
|
CanaryService: r.conf.CanaryService,
|
||||||
StableService: r.conf.StableService,
|
StableService: r.conf.StableService,
|
||||||
PatchPodMetadata: rollout.Spec.Strategy.Canary.PatchPodTemplateMetadata,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data)
|
unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data)
|
||||||
|
|
|
||||||
|
|
@ -59,7 +59,7 @@ var (
|
||||||
"route": [
|
"route": [
|
||||||
{
|
{
|
||||||
"destination": {
|
"destination": {
|
||||||
"host": "echoserver",
|
"host": "echoserver"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
@ -111,7 +111,7 @@ func TestInitialize(t *testing.T) {
|
||||||
Namespace: util.GetRolloutNamespace(),
|
Namespace: util.GetRolloutNamespace(),
|
||||||
},
|
},
|
||||||
Data: map[string]string{
|
Data: map[string]string{
|
||||||
fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript",
|
fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingCustomTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
@ -131,7 +131,7 @@ func TestInitialize(t *testing.T) {
|
||||||
getUnstructured: func() *unstructured.Unstructured {
|
getUnstructured: func() *unstructured.Unstructured {
|
||||||
u := &unstructured.Unstructured{}
|
u := &unstructured.Unstructured{}
|
||||||
_ = u.UnmarshalJSON([]byte(networkDemo))
|
_ = u.UnmarshalJSON([]byte(networkDemo))
|
||||||
u.SetAPIVersion("networking.test.io/v1alpha3")
|
u.SetAPIVersion("networking.istio.io/v1alpha3")
|
||||||
return u
|
return u
|
||||||
},
|
},
|
||||||
getConfig: func() Config {
|
getConfig: func() Config {
|
||||||
|
|
@ -140,7 +140,7 @@ func TestInitialize(t *testing.T) {
|
||||||
CanaryService: "echoserver-canary",
|
CanaryService: "echoserver-canary",
|
||||||
TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{
|
TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{
|
||||||
{
|
{
|
||||||
APIVersion: "networking.test.io/v1alpha3",
|
APIVersion: "networking.istio.io/v1alpha3",
|
||||||
Kind: "VirtualService",
|
Kind: "VirtualService",
|
||||||
Name: "echoserver",
|
Name: "echoserver",
|
||||||
},
|
},
|
||||||
|
|
@ -154,14 +154,14 @@ func TestInitialize(t *testing.T) {
|
||||||
Namespace: util.GetRolloutNamespace(),
|
Namespace: util.GetRolloutNamespace(),
|
||||||
},
|
},
|
||||||
Data: map[string]string{
|
Data: map[string]string{
|
||||||
fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.test.io"): "ExpectedLuaScript",
|
fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
expectUnstructured: func() *unstructured.Unstructured {
|
expectUnstructured: func() *unstructured.Unstructured {
|
||||||
u := &unstructured.Unstructured{}
|
u := &unstructured.Unstructured{}
|
||||||
_ = u.UnmarshalJSON([]byte(networkDemo))
|
_ = u.UnmarshalJSON([]byte(networkDemo))
|
||||||
u.SetAPIVersion("networking.test.io/v1alpha3")
|
u.SetAPIVersion("networking.istio.io/v1alpha3")
|
||||||
annotations := map[string]string{
|
annotations := map[string]string{
|
||||||
OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`,
|
OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`,
|
||||||
"virtual": "test",
|
"virtual": "test",
|
||||||
|
|
@ -227,22 +227,18 @@ func TestEnsureRoutes(t *testing.T) {
|
||||||
getUnstructured: func() *unstructured.Unstructured {
|
getUnstructured: func() *unstructured.Unstructured {
|
||||||
u := &unstructured.Unstructured{}
|
u := &unstructured.Unstructured{}
|
||||||
_ = u.UnmarshalJSON([]byte(networkDemo))
|
_ = u.UnmarshalJSON([]byte(networkDemo))
|
||||||
annotations := map[string]string{
|
u.SetAPIVersion("networking.istio.io/v1alpha3")
|
||||||
OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`,
|
|
||||||
"virtual": "test",
|
|
||||||
}
|
|
||||||
u.SetAnnotations(annotations)
|
|
||||||
return u
|
return u
|
||||||
},
|
},
|
||||||
expectInfo: func() (bool, *unstructured.Unstructured) {
|
expectInfo: func() (bool, *unstructured.Unstructured) {
|
||||||
u := &unstructured.Unstructured{}
|
u := &unstructured.Unstructured{}
|
||||||
_ = u.UnmarshalJSON([]byte(networkDemo))
|
_ = u.UnmarshalJSON([]byte(networkDemo))
|
||||||
annotations := map[string]string{
|
annotations := map[string]string{
|
||||||
OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}}}]}]},"annotations":{"virtual":"test"}}`,
|
OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`,
|
||||||
"virtual": "test",
|
"virtual": "test",
|
||||||
}
|
}
|
||||||
u.SetAnnotations(annotations)
|
u.SetAnnotations(annotations)
|
||||||
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}},"weight":95},{"destination":{"host":"echoserver-canary","port":{"number":80}},"weight":5}]}]}`
|
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}]}]}`
|
||||||
var spec interface{}
|
var spec interface{}
|
||||||
_ = json.Unmarshal([]byte(specStr), &spec)
|
_ = json.Unmarshal([]byte(specStr), &spec)
|
||||||
u.Object["spec"] = spec
|
u.Object["spec"] = spec
|
||||||
|
|
@ -251,7 +247,7 @@ func TestEnsureRoutes(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
config := Config{
|
config := Config{
|
||||||
RolloutName: "rollout-demo",
|
Key: "rollout-demo",
|
||||||
StableService: "echoserver",
|
StableService: "echoserver",
|
||||||
CanaryService: "echoserver-canary",
|
CanaryService: "echoserver-canary",
|
||||||
TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{
|
TrafficConf: []rolloutsv1alpha1.CustomNetworkRef{
|
||||||
|
|
@ -298,11 +294,11 @@ func TestFinalise(t *testing.T) {
|
||||||
u := &unstructured.Unstructured{}
|
u := &unstructured.Unstructured{}
|
||||||
_ = u.UnmarshalJSON([]byte(networkDemo))
|
_ = u.UnmarshalJSON([]byte(networkDemo))
|
||||||
annotations := map[string]string{
|
annotations := map[string]string{
|
||||||
OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`,
|
OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`,
|
||||||
"virtual": "test",
|
"virtual": "test",
|
||||||
}
|
}
|
||||||
u.SetAnnotations(annotations)
|
u.SetAnnotations(annotations)
|
||||||
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":100},{"destination":{"host":"echoserver-canary"},"weight":0}}]}]}`
|
specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}}]}]}`
|
||||||
var spec interface{}
|
var spec interface{}
|
||||||
_ = json.Unmarshal([]byte(specStr), &spec)
|
_ = json.Unmarshal([]byte(specStr), &spec)
|
||||||
u.Object["spec"] = spec
|
u.Object["spec"] = spec
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,225 @@
|
||||||
|
-- obj = { canaryWeight = 20, stableWeight = 80,
|
||||||
|
-- matches = {
|
||||||
|
-- { headers = { { name = "user-agent", value = "pc", type = "Exact", },
|
||||||
|
-- { type = "RegularExpression", name = "name", value = ".*demo", }, }, }, },
|
||||||
|
-- canaryService = "nginx-service-canary", stableService = "nginx-service",
|
||||||
|
-- data = {
|
||||||
|
-- spec = { hosts = { "*", }, http = { { route = { { destination = { host = "nginx-service", subset = "v1"}, }, { destination = { host = "nginx-service", subset = "v2"}, } }, }, },
|
||||||
|
-- gateways = { "nginx-gateway", }, }, }, }
|
||||||
|
|
||||||
|
spec = obj.data.spec
|
||||||
|
|
||||||
|
if obj.canaryWeight == -1 then
|
||||||
|
obj.canaryWeight = 100
|
||||||
|
obj.stableWeight = 0
|
||||||
|
end
|
||||||
|
|
||||||
|
function FindRules(spec, protocol)
|
||||||
|
local rules = {}
|
||||||
|
if (protocol == "http") then
|
||||||
|
if (spec.http ~= nil) then
|
||||||
|
for _, http in ipairs(spec.http) do
|
||||||
|
table.insert(rules, http)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
elseif (protocol == "tcp") then
|
||||||
|
if (spec.tcp ~= nil) then
|
||||||
|
for _, http in ipairs(spec.tcp) do
|
||||||
|
table.insert(rules, http)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
elseif (protocol == "tls") then
|
||||||
|
if (spec.tls ~= nil) then
|
||||||
|
for _, http in ipairs(spec.tls) do
|
||||||
|
table.insert(rules, http)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return rules
|
||||||
|
end
|
||||||
|
|
||||||
|
-- find matched route of VirtualService spec with stable svc
|
||||||
|
function FindMatchedRules(spec, stableService, protocol)
|
||||||
|
local matchedRoutes = {}
|
||||||
|
local rules = FindRules(spec, protocol)
|
||||||
|
-- a rule contains 'match' and 'route'
|
||||||
|
for _, rule in ipairs(rules) do
|
||||||
|
for _, route in ipairs(rule.route) do
|
||||||
|
if route.destination.host == stableService then
|
||||||
|
table.insert(matchedRoutes, rule)
|
||||||
|
break
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return matchedRoutes
|
||||||
|
end
|
||||||
|
|
||||||
|
function FindStableServiceSubsets(spec, stableService, protocol)
|
||||||
|
local stableSubsets = {}
|
||||||
|
local rules = FindRules(spec, protocol)
|
||||||
|
local hasRule = false
|
||||||
|
-- a rule contains 'match' and 'route'
|
||||||
|
for _, rule in ipairs(rules) do
|
||||||
|
for _, route in ipairs(rule.route) do
|
||||||
|
if route.destination.host == stableService then
|
||||||
|
hasRule = true
|
||||||
|
local contains = false
|
||||||
|
for _, v in ipairs(stableSubsets) do
|
||||||
|
if v == route.destination.subset then
|
||||||
|
contains = true
|
||||||
|
break
|
||||||
|
end
|
||||||
|
end
|
||||||
|
if not contains and route.destination.subset ~= nil then
|
||||||
|
table.insert(stableSubsets, route.destination.subset)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return hasRule, stableSubsets
|
||||||
|
end
|
||||||
|
|
||||||
|
function DeepCopy(original)
|
||||||
|
local copy
|
||||||
|
if type(original) == 'table' then
|
||||||
|
copy = {}
|
||||||
|
for key, value in pairs(original) do
|
||||||
|
copy[key] = DeepCopy(value)
|
||||||
|
end
|
||||||
|
else
|
||||||
|
copy = original
|
||||||
|
end
|
||||||
|
return copy
|
||||||
|
end
|
||||||
|
|
||||||
|
function CalculateWeight(route, stableWeight, n)
|
||||||
|
local weight
|
||||||
|
if (route.weight) then
|
||||||
|
weight = math.floor(route.weight * stableWeight / 100)
|
||||||
|
else
|
||||||
|
weight = math.floor(stableWeight / n)
|
||||||
|
end
|
||||||
|
return weight
|
||||||
|
end
|
||||||
|
|
||||||
|
-- generate routes with matches, insert a rule before other rules
|
||||||
|
function GenerateMatchedRoutes(spec, matches, stableService, canaryService, stableWeight, canaryWeight, protocol)
|
||||||
|
local hasRule, stableServiceSubsets = FindStableServiceSubsets(spec, stableService, protocol)
|
||||||
|
if (not hasRule) then
|
||||||
|
return
|
||||||
|
end
|
||||||
|
for _, match in ipairs(matches) do
|
||||||
|
local route = {}
|
||||||
|
route["match"] = {}
|
||||||
|
|
||||||
|
for key, value in pairs(match) do
|
||||||
|
local vsMatch = {}
|
||||||
|
vsMatch[key] = {}
|
||||||
|
for _, rule in ipairs(value) do
|
||||||
|
if rule["type"] == "RegularExpression" then
|
||||||
|
matchType = "regex"
|
||||||
|
elseif rule["type"] == "Exact" then
|
||||||
|
matchType = "exact"
|
||||||
|
elseif rule["type"] == "Prefix" then
|
||||||
|
matchType = "prefix"
|
||||||
|
end
|
||||||
|
if key == "headers" then
|
||||||
|
vsMatch[key][rule["name"]] = {}
|
||||||
|
vsMatch[key][rule["name"]][matchType] = rule.value
|
||||||
|
else
|
||||||
|
vsMatch[key][matchType] = rule.value
|
||||||
|
end
|
||||||
|
end
|
||||||
|
table.insert(route["match"], vsMatch)
|
||||||
|
end
|
||||||
|
route.route = {
|
||||||
|
{
|
||||||
|
destination = {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
-- if stableWeight != 0, then add stable service destinations
|
||||||
|
-- incase there are multiple subsets in stable service
|
||||||
|
if stableWeight ~= 0 then
|
||||||
|
local nRoute = {}
|
||||||
|
if #stableServiceSubsets ~= 0 then
|
||||||
|
local weight = CalculateWeight(nRoute, stableWeight, #stableServiceSubsets)
|
||||||
|
for _, r in ipairs(stableServiceSubsets) do
|
||||||
|
nRoute = {
|
||||||
|
destination = {
|
||||||
|
host = stableService,
|
||||||
|
subset = r
|
||||||
|
},
|
||||||
|
weight = weight
|
||||||
|
}
|
||||||
|
table.insert(route.route, nRoute)
|
||||||
|
end
|
||||||
|
else
|
||||||
|
nRoute = {
|
||||||
|
destination = {
|
||||||
|
host = stableService
|
||||||
|
},
|
||||||
|
weight = stableWeight
|
||||||
|
}
|
||||||
|
table.insert(route.route, nRoute)
|
||||||
|
end
|
||||||
|
-- update every matched route
|
||||||
|
route.route[1].weight = canaryWeight
|
||||||
|
end
|
||||||
|
-- if stableService == canaryService, then do e2e release
|
||||||
|
if stableService == canaryService then
|
||||||
|
route.route[1].destination.host = stableService
|
||||||
|
route.route[1].destination.subset = "canary"
|
||||||
|
else
|
||||||
|
route.route[1].destination.host = canaryService
|
||||||
|
end
|
||||||
|
if (protocol == "http") then
|
||||||
|
table.insert(spec.http, 1, route)
|
||||||
|
elseif (protocol == "tls") then
|
||||||
|
table.insert(spec.tls, 1, route)
|
||||||
|
elseif (protocol == "tcp") then
|
||||||
|
table.insert(spec.tcp, 1, route)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
-- generate routes without matches, change every rule
|
||||||
|
function GenerateRoutes(spec, stableService, canaryService, stableWeight, canaryWeight, protocol)
|
||||||
|
local matchedRules = FindMatchedRules(spec, stableService, protocol)
|
||||||
|
for _, rule in ipairs(matchedRules) do
|
||||||
|
local canary
|
||||||
|
if stableService ~= canaryService then
|
||||||
|
canary = {
|
||||||
|
destination = {
|
||||||
|
host = canaryService,
|
||||||
|
},
|
||||||
|
weight = canaryWeight,
|
||||||
|
}
|
||||||
|
else
|
||||||
|
canary = {
|
||||||
|
destination = {
|
||||||
|
host = stableService,
|
||||||
|
subset = "canary",
|
||||||
|
},
|
||||||
|
weight = canaryWeight,
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
-- incase there are multiple versions traffic already, do a for-loop
|
||||||
|
for _, route in ipairs(rule.route) do
|
||||||
|
-- update stable service weight
|
||||||
|
route.weight = CalculateWeight(route, stableWeight, #rule.route)
|
||||||
|
end
|
||||||
|
table.insert(rule.route, canary)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if (obj.matches) then
|
||||||
|
GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "http")
|
||||||
|
GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tcp")
|
||||||
|
GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tls")
|
||||||
|
else
|
||||||
|
GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "http")
|
||||||
|
GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tcp")
|
||||||
|
GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tls")
|
||||||
|
end
|
||||||
|
return obj.data
|
||||||
|
|
@ -19,6 +19,7 @@ package luamanager
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
|
rolloutv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
|
||||||
|
|
@ -26,10 +27,33 @@ import (
|
||||||
lua "github.com/yuin/gopher-lua"
|
lua "github.com/yuin/gopher-lua"
|
||||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
"k8s.io/apimachinery/pkg/runtime"
|
"k8s.io/apimachinery/pkg/runtime"
|
||||||
|
utilpointer "k8s.io/utils/pointer"
|
||||||
luajson "layeh.com/gopher-json"
|
luajson "layeh.com/gopher-json"
|
||||||
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
|
gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
|
||||||
|
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"sigs.k8s.io/yaml"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type LuaData struct {
|
||||||
|
Data Data
|
||||||
|
CanaryWeight int32
|
||||||
|
StableWeight int32
|
||||||
|
Matches []rolloutv1alpha1.HttpRouteMatch
|
||||||
|
CanaryService string
|
||||||
|
StableService string
|
||||||
|
RevisionLabelKey string
|
||||||
|
StableRevision string
|
||||||
|
CanaryRevision string
|
||||||
|
}
|
||||||
|
type Data struct {
|
||||||
|
Spec interface{} `json:"spec,omitempty"`
|
||||||
|
Labels map[string]string `json:"labels,omitempty"`
|
||||||
|
Annotations map[string]string `json:"annotations,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunLuaScript(t *testing.T) {
|
func TestRunLuaScript(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
|
|
@ -138,3 +162,176 @@ func TestRunLuaScript(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type LuaTestCase struct {
|
||||||
|
Matches []rolloutv1alpha1.HttpRouteMatch `yaml:"matches"`
|
||||||
|
StableService string `yaml:"stableService"`
|
||||||
|
CanaryService string `yaml:"canaryService"`
|
||||||
|
StableWeight int `yaml:"stableWeight"`
|
||||||
|
CanaryWeight int `yaml:"canaryWeight"`
|
||||||
|
Spec interface{} `yaml:"spec"`
|
||||||
|
NSpec interface{} `yaml:"nSpec"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type TestCase struct {
|
||||||
|
Rollout *rolloutv1alpha1.Rollout `json:"rollout,omitempty"`
|
||||||
|
TrafficRouting *rolloutv1alpha1.TrafficRouting `json:"trafficRouting,omitempty"`
|
||||||
|
Original *unstructured.Unstructured `json:"original,omitempty"`
|
||||||
|
Expected []*unstructured.Unstructured `json:"expected,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// test if the lua script run as expected
|
||||||
|
func TestLuaScript(t *testing.T) {
|
||||||
|
err := filepath.Walk("../../../lua_configuration", func(path string, f os.FileInfo, err error) error {
|
||||||
|
if !strings.Contains(path, "trafficRouting.lua") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
script, err := readScript(t, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
dir := filepath.Dir(path)
|
||||||
|
err = filepath.Walk(filepath.Join(dir, "testdata"), func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !info.IsDir() && filepath.Ext(path) == ".yaml" || filepath.Ext(path) == ".yml" {
|
||||||
|
testCase := getLuaTestCase(t, path)
|
||||||
|
rollout := testCase.Rollout
|
||||||
|
trafficRouting := testCase.TrafficRouting
|
||||||
|
if rollout != nil {
|
||||||
|
steps := rollout.Spec.Strategy.Canary.Steps
|
||||||
|
for i, step := range steps {
|
||||||
|
weight := step.TrafficRoutingStrategy.Weight
|
||||||
|
if weight == nil {
|
||||||
|
weight = utilpointer.Int32(-1)
|
||||||
|
}
|
||||||
|
var canaryService string
|
||||||
|
stableService := rollout.Spec.Strategy.Canary.TrafficRoutings[0].Service
|
||||||
|
canaryService = fmt.Sprintf("%s-canary", stableService)
|
||||||
|
data := &LuaData{
|
||||||
|
Data: Data{
|
||||||
|
Labels: testCase.Original.GetLabels(),
|
||||||
|
Annotations: testCase.Original.GetAnnotations(),
|
||||||
|
Spec: testCase.Original.Object["spec"],
|
||||||
|
},
|
||||||
|
Matches: step.TrafficRoutingStrategy.Matches,
|
||||||
|
CanaryWeight: *weight,
|
||||||
|
StableWeight: 100 - *weight,
|
||||||
|
CanaryService: canaryService,
|
||||||
|
StableService: stableService,
|
||||||
|
RevisionLabelKey: "pod-template-hash",
|
||||||
|
StableRevision: "pod-template-hash-stable",
|
||||||
|
CanaryRevision: "pod-template-hash-canary",
|
||||||
|
}
|
||||||
|
nSpec, err := executeLua(data, script)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
eSpec := Data{
|
||||||
|
Spec: testCase.Expected[i].Object["spec"],
|
||||||
|
Annotations: testCase.Expected[i].GetAnnotations(),
|
||||||
|
Labels: testCase.Expected[i].GetLabels(),
|
||||||
|
}
|
||||||
|
if util.DumpJSON(eSpec) != util.DumpJSON(nSpec) {
|
||||||
|
return fmt.Errorf("expect %s, but get %s", util.DumpJSON(eSpec), util.DumpJSON(nSpec))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if trafficRouting != nil {
|
||||||
|
weight := trafficRouting.Spec.Strategy.Weight
|
||||||
|
if weight == nil {
|
||||||
|
weight = utilpointer.Int32(-1)
|
||||||
|
}
|
||||||
|
var canaryService string
|
||||||
|
stableService := trafficRouting.Spec.ObjectRef[0].Service
|
||||||
|
canaryService = stableService
|
||||||
|
data := &LuaData{
|
||||||
|
Data: Data{
|
||||||
|
Labels: testCase.Original.GetLabels(),
|
||||||
|
Annotations: testCase.Original.GetAnnotations(),
|
||||||
|
Spec: testCase.Original.Object["spec"],
|
||||||
|
},
|
||||||
|
Matches: trafficRouting.Spec.Strategy.Matches,
|
||||||
|
CanaryWeight: *weight,
|
||||||
|
StableWeight: 100 - *weight,
|
||||||
|
CanaryService: canaryService,
|
||||||
|
StableService: stableService,
|
||||||
|
RevisionLabelKey: "pod-template-hash",
|
||||||
|
StableRevision: "pod-template-hash-stable",
|
||||||
|
CanaryRevision: "pod-template-hash-canary",
|
||||||
|
}
|
||||||
|
nSpec, err := executeLua(data, script)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
eSpec := Data{
|
||||||
|
Spec: testCase.Expected[0].Object["spec"],
|
||||||
|
Annotations: testCase.Expected[0].GetAnnotations(),
|
||||||
|
Labels: testCase.Expected[0].GetLabels(),
|
||||||
|
}
|
||||||
|
if util.DumpJSON(eSpec) != util.DumpJSON(nSpec) {
|
||||||
|
return fmt.Errorf("expect %s, but get %s", util.DumpJSON(eSpec), util.DumpJSON(nSpec))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("neither rollout nor trafficRouting defined in test case: %s", path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to test lua scripts: %s", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readScript(t *testing.T, path string) (string, error) {
|
||||||
|
data, err := os.ReadFile(filepath.Clean(path))
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return string(data), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func getLuaTestCase(t *testing.T, path string) *TestCase {
|
||||||
|
yamlFile, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to read file %s", path)
|
||||||
|
}
|
||||||
|
luaTestCase := &TestCase{}
|
||||||
|
err = yaml.Unmarshal(yamlFile, luaTestCase)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("test case %s format error", path)
|
||||||
|
}
|
||||||
|
return luaTestCase
|
||||||
|
}
|
||||||
|
|
||||||
|
func executeLua(data *LuaData, script string) (Data, error) {
|
||||||
|
luaManager := &LuaManager{}
|
||||||
|
unObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(data)
|
||||||
|
if err != nil {
|
||||||
|
return Data{}, err
|
||||||
|
}
|
||||||
|
u := &unstructured.Unstructured{Object: unObj}
|
||||||
|
l, err := luaManager.RunLuaScript(u, script)
|
||||||
|
if err != nil {
|
||||||
|
return Data{}, err
|
||||||
|
}
|
||||||
|
returnValue := l.Get(-1)
|
||||||
|
var nSpec Data
|
||||||
|
if returnValue.Type() == lua.LTTable {
|
||||||
|
jsonBytes, err := luajson.Encode(returnValue)
|
||||||
|
if err != nil {
|
||||||
|
return Data{}, err
|
||||||
|
}
|
||||||
|
err = json.Unmarshal(jsonBytes, &nSpec)
|
||||||
|
if err != nil {
|
||||||
|
return Data{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nSpec, nil
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue