add unit tests and e2e tests

Signed-off-by: Kuromesi <blackfacepan@163.com>

config/crd/bases/rollouts.kruise.io_rollouts.yaml

@@ -341,7 +341,7 @@ spec:
                             traffic routing
                           properties:
                             createCanaryService:
-                              default: false
+                              default: true
                               description: create a new canary service or just use
                                 the stable service
                               type: boolean

config/crd/bases/rollouts.kruise.io_trafficroutings.yaml

@@ -55,7 +55,7 @@ spec:
                     routing
                   properties:
                     createCanaryService:
-                      default: false
+                      default: true
                       description: create a new canary service or just use the stable
                         service
                       type: boolean

pkg/controller/trafficrouting/trafficrouting_controller_test.go

@@ -116,7 +116,8 @@ var (
 		Spec: v1alpha1.TrafficRoutingSpec{
 			ObjectRef: []v1alpha1.TrafficRoutingRef{
 				{
-					Service: "echoserver",
+					Service:             "echoserver",
+					CreateCanaryService: false,
 					Ingress: &v1alpha1.IngressTrafficRouting{
 						Name: "echoserver",
 					},
@@ -379,6 +380,7 @@ func TestTrafficRoutingTest(t *testing.T) {
 			for _, obj := range ig {
 				checkObjEqual(client, t, obj)
 			}
+			manager.trafficRoutingManager.RemoveTrafficRoutingController(newTrafficRoutingContext(tr))
 		})
 	}
 }

pkg/trafficrouting/manager_test.go

@@ -666,6 +666,7 @@ func TestFinalisingTrafficRouting(t *testing.T) {
 			}
 			manager := NewTrafficRoutingManager(client)
 			done, err := manager.FinalisingTrafficRouting(c, cs.onlyRestoreStableService)
+			manager.RemoveTrafficRoutingController(c)
 			if err != nil {
 				t.Fatalf("DoTrafficRouting failed: %s", err)
 			}

pkg/trafficrouting/network/custom/custom.go

@@ -188,10 +188,8 @@ func (r *customController) storeObject(obj *unstructured.Unstructured) error {
 	}
 	annotations[OriginalSpecAnnotation] = cSpec
 	obj.SetAnnotations(annotations)
-	if err := r.Update(context.TODO(), obj); err != nil {
-		return err
-	}
-	return nil
+	err := r.Update(context.TODO(), obj)
+	return err
 }
 
 // restore an object from spec stored in OriginalSpecAnnotation
@@ -206,10 +204,8 @@ func (r *customController) restoreObject(obj *unstructured.Unstructured) error {
 	obj.Object["spec"] = oSpec.Spec
 	obj.SetAnnotations(oSpec.Annotations)
 	obj.SetLabels(oSpec.Labels)
-	if err := r.Update(context.TODO(), obj); err != nil {
-		return err
-	}
-	return nil
+	err := r.Update(context.TODO(), obj)
+	return err
 }
 
 func (r *customController) executeLuaForCanary(spec Data, strategy *rolloutv1alpha1.TrafficRoutingStrategy, luaScript string) (Data, error) {

pkg/trafficrouting/network/custom/custom_test.go

@@ -0,0 +1,348 @@
+/*
+Copyright 2021.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package custom
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"testing"
+
+	rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
+	"github.com/openkruise/rollouts/pkg/util"
+	"github.com/openkruise/rollouts/pkg/util/configuration"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/klog/v2"
+	utilpointer "k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+var (
+	scheme      *runtime.Scheme
+	networkDemo = `
+						{
+							"apiVersion": "networking.istio.io/v1alpha3",
+							"kind": "VirtualService",
+							"metadata": {
+								"name": "echoserver",
+								"annotations": {
+									"virtual": "test"
+								}
+							},
+							"spec": {
+								"hosts": [
+									"echoserver.example.com"
+								],
+								"http": [
+									{
+										"route": [
+											{
+												"destination": {
+													"host": "echoserver",
+												}
+											}
+										]
+									}
+								]
+							}
+						}
+						`
+)
+
+func init() {
+	scheme = runtime.NewScheme()
+	_ = clientgoscheme.AddToScheme(scheme)
+	_ = rolloutsv1alpha1.AddToScheme(scheme)
+}
+
+func TestInitialize(t *testing.T) {
+	cases := []struct {
+		name               string
+		getUnstructured    func() *unstructured.Unstructured
+		getConfig          func() Config
+		getConfigMap       func() *corev1.ConfigMap
+		expectUnstructured func() *unstructured.Unstructured
+	}{
+		{
+			name: "test1, find lua script locally",
+			getUnstructured: func() *unstructured.Unstructured {
+				u := &unstructured.Unstructured{}
+				_ = u.UnmarshalJSON([]byte(networkDemo))
+				return u
+			},
+			getConfig: func() Config {
+				return Config{
+					StableService: "echoserver",
+					CanaryService: "echoserver-canary",
+					TrafficConf: []rolloutsv1alpha1.NetworkRef{
+						{
+							APIVersion: "networking.istio.io/v1alpha3",
+							Kind:       "VirtualService",
+							Name:       "echoserver",
+						},
+					},
+				}
+			},
+			getConfigMap: func() *corev1.ConfigMap {
+				return &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      LuaConfigMap,
+						Namespace: util.GetRolloutNamespace(),
+					},
+					Data: map[string]string{
+						fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.istio.io"): "ExpectedLuaScript",
+					},
+				}
+			},
+			expectUnstructured: func() *unstructured.Unstructured {
+				u := &unstructured.Unstructured{}
+				_ = u.UnmarshalJSON([]byte(networkDemo))
+				annotations := map[string]string{
+					OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`,
+					"virtual":              "test",
+				}
+				u.SetAnnotations(annotations)
+				return u
+			},
+		},
+		{
+			name: "test2, find lua script in ConfigMap",
+			getUnstructured: func() *unstructured.Unstructured {
+				u := &unstructured.Unstructured{}
+				_ = u.UnmarshalJSON([]byte(networkDemo))
+				u.SetAPIVersion("networking.test.io/v1alpha3")
+				return u
+			},
+			getConfig: func() Config {
+				return Config{
+					StableService: "echoserver",
+					CanaryService: "echoserver-canary",
+					TrafficConf: []rolloutsv1alpha1.NetworkRef{
+						{
+							APIVersion: "networking.test.io/v1alpha3",
+							Kind:       "VirtualService",
+							Name:       "echoserver",
+						},
+					},
+				}
+			},
+			getConfigMap: func() *corev1.ConfigMap {
+				return &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      LuaConfigMap,
+						Namespace: util.GetRolloutNamespace(),
+					},
+					Data: map[string]string{
+						fmt.Sprintf("%s.%s.%s", configuration.LuaTrafficRoutingIngressTypePrefix, "VirtualService", "networking.test.io"): "ExpectedLuaScript",
+					},
+				}
+			},
+			expectUnstructured: func() *unstructured.Unstructured {
+				u := &unstructured.Unstructured{}
+				_ = u.UnmarshalJSON([]byte(networkDemo))
+				u.SetAPIVersion("networking.test.io/v1alpha3")
+				annotations := map[string]string{
+					OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`,
+					"virtual":              "test",
+				}
+				u.SetAnnotations(annotations)
+				return u
+			},
+		},
+	}
+
+	for _, cs := range cases {
+		t.Run(cs.name, func(t *testing.T) {
+			fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build()
+			err := fakeCli.Create(context.TODO(), cs.getUnstructured())
+			if err != nil {
+				klog.Errorf(err.Error())
+				return
+			}
+			if err := fakeCli.Create(context.TODO(), cs.getConfigMap()); err != nil {
+				klog.Errorf(err.Error())
+			}
+			c, _ := NewCustomController(fakeCli, cs.getConfig())
+			err = c.Initialize(context.TODO())
+			if err != nil {
+				t.Fatalf("Initialize failed: %s", err.Error())
+			}
+			checkEqual(fakeCli, t, cs.expectUnstructured())
+		})
+	}
+}
+
+func checkEqual(cli client.Client, t *testing.T, expect *unstructured.Unstructured) {
+	obj := &unstructured.Unstructured{}
+	obj.SetAPIVersion(expect.GetAPIVersion())
+	obj.SetKind(expect.GetKind())
+	if err := cli.Get(context.TODO(), types.NamespacedName{Namespace: expect.GetNamespace(), Name: expect.GetName()}, obj); err != nil {
+		t.Fatalf("Get object failed: %s", err.Error())
+	}
+	if !reflect.DeepEqual(obj.GetAnnotations(), expect.GetAnnotations()) {
+		fmt.Println(util.DumpJSON(obj.GetAnnotations()), util.DumpJSON(expect.GetAnnotations()))
+		t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.GetAnnotations()), util.DumpJSON(obj.GetAnnotations()))
+	}
+	if util.DumpJSON(expect.Object["spec"]) != util.DumpJSON(obj.Object["spec"]) {
+		t.Fatalf("expect(%s), but get(%s)", util.DumpJSON(expect.Object["spec"]), util.DumpJSON(obj.Object["spec"]))
+	}
+}
+
+func TestEnsureRoutes(t *testing.T) {
+	cases := []struct {
+		name            string
+		getLua          func() map[string]string
+		getRoutes       func() *rolloutsv1alpha1.TrafficRoutingStrategy
+		getUnstructured func() *unstructured.Unstructured
+		expectInfo      func() (bool, *unstructured.Unstructured)
+	}{
+		{
+			name: "test1",
+			getRoutes: func() *rolloutsv1alpha1.TrafficRoutingStrategy {
+				return &rolloutsv1alpha1.TrafficRoutingStrategy{
+					Weight: utilpointer.Int32(5),
+				}
+			},
+			getUnstructured: func() *unstructured.Unstructured {
+				u := &unstructured.Unstructured{}
+				_ = u.UnmarshalJSON([]byte(networkDemo))
+				annotations := map[string]string{
+					OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]},"annotations":{"virtual":"test"}}`,
+					"virtual":              "test",
+				}
+				u.SetAnnotations(annotations)
+				return u
+			},
+			expectInfo: func() (bool, *unstructured.Unstructured) {
+				u := &unstructured.Unstructured{}
+				_ = u.UnmarshalJSON([]byte(networkDemo))
+				annotations := map[string]string{
+					OriginalSpecAnnotation: `{"spec":{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}}}]}]},"annotations":{"virtual":"test"}}`,
+					"virtual":              "test",
+				}
+				u.SetAnnotations(annotations)
+				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver","port":{"number":80}},"weight":95},{"destination":{"host":"echoserver-canary","port":{"number":80}},"weight":5}]}]}`
+				var spec interface{}
+				_ = json.Unmarshal([]byte(specStr), &spec)
+				u.Object["spec"] = spec
+				return false, u
+			},
+		},
+	}
+	config := Config{
+		RolloutName:   "rollout-demo",
+		StableService: "echoserver",
+		CanaryService: "echoserver-canary",
+		TrafficConf: []rolloutsv1alpha1.NetworkRef{
+			{
+				APIVersion: "networking.istio.io/v1alpha3",
+				Kind:       "VirtualService",
+				Name:       "echoserver",
+			},
+		},
+	}
+	for _, cs := range cases {
+		t.Run(cs.name, func(t *testing.T) {
+			fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build()
+			err := fakeCli.Create(context.TODO(), cs.getUnstructured())
+			if err != nil {
+				klog.Errorf(err.Error())
+				return
+			}
+			c, _ := NewCustomController(fakeCli, config)
+			strategy := cs.getRoutes()
+			expect1, expect2 := cs.expectInfo()
+			c.Initialize(context.TODO())
+			done, err := c.EnsureRoutes(context.TODO(), strategy)
+			if err != nil {
+				t.Fatalf("EnsureRoutes failed: %s", err.Error())
+			} else if done != expect1 {
+				t.Fatalf("expect(%v), but get(%v)", expect1, done)
+			}
+			checkEqual(fakeCli, t, expect2)
+		})
+	}
+}
+
+func TestFinalise(t *testing.T) {
+	cases := []struct {
+		name               string
+		getUnstructured    func() *unstructured.Unstructured
+		getConfig          func() Config
+		expectUnstructured func() *unstructured.Unstructured
+	}{
+		{
+			name: "test1",
+			getUnstructured: func() *unstructured.Unstructured {
+				u := &unstructured.Unstructured{}
+				_ = u.UnmarshalJSON([]byte(networkDemo))
+				annotations := map[string]string{
+					OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`,
+					"virtual":              "test",
+				}
+				u.SetAnnotations(annotations)
+				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":100},{"destination":{"host":"echoserver-canary"},"weight":0}}]}]}`
+				var spec interface{}
+				_ = json.Unmarshal([]byte(specStr), &spec)
+				u.Object["spec"] = spec
+				return u
+			},
+			getConfig: func() Config {
+				return Config{
+					StableService: "echoserver",
+					CanaryService: "echoserver-canary",
+					TrafficConf: []rolloutsv1alpha1.NetworkRef{
+						{
+							APIVersion: "networking.istio.io/v1alpha3",
+							Kind:       "VirtualService",
+							Name:       "echoserver",
+						},
+					},
+				}
+			},
+			expectUnstructured: func() *unstructured.Unstructured {
+				u := &unstructured.Unstructured{}
+				_ = u.UnmarshalJSON([]byte(networkDemo))
+				return u
+			},
+		},
+	}
+
+	for _, cs := range cases {
+		t.Run(cs.name, func(t *testing.T) {
+			fakeCli := fake.NewClientBuilder().WithScheme(scheme).Build()
+			err := fakeCli.Create(context.TODO(), cs.getUnstructured())
+			if err != nil {
+				klog.Errorf(err.Error())
+				return
+			}
+			c, _ := NewCustomController(fakeCli, cs.getConfig())
+			err = c.Finalise(context.TODO())
+			if err != nil {
+				t.Fatalf("Initialize failed: %s", err.Error())
+			}
+			checkEqual(fakeCli, t, cs.expectUnstructured())
+		})
+	}
+}

test/e2e/rollout_test.go

@@ -19,6 +19,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"sort"
 	"strings"
 	"time"
@@ -34,6 +35,7 @@ import (
 	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/util/retry"
@@ -5582,6 +5584,51 @@ var _ = SIGDescribe("Rollout", func() {
 			Expect(rollout1.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy))
 		})
 	})
+
+	KruiseDescribe("Custom network provider tests", func() {
+		It("Istio VirtualService test", func() {
+			index1 := &v1.ConfigMap{}
+			index2 := &v1.ConfigMap{}
+			Expect(ReadYamlToObject("./test_data/custom/index1.yaml", index1)).ToNot(HaveOccurred())
+			Expect(ReadYamlToObject("./test_data/custom/index2.yaml", index2)).ToNot(HaveOccurred())
+			CreateObject(index1)
+			CreateObject(index2)
+
+			svc := &v1.Service{}
+			Expect(ReadYamlToObject("./test_data/custom/service.yaml", svc)).ToNot(HaveOccurred())
+			CreateObject(svc)
+
+			app := &apps.Deployment{}
+			Expect(ReadYamlToObject("./test_data/custom/appv1.yaml", app)).ToNot(HaveOccurred())
+			CreateObject(app)
+			WaitDeploymentAllPodsReady(app)
+
+			virtualService := &unstructured.Unstructured{}
+			Expect(ReadYamlToObject("./test_data/custom/virtualService.yaml", virtualService)).ToNot(HaveOccurred())
+			CreateObject(virtualService)
+			expectVirtualService := &unstructured.Unstructured{}
+
+			rollout := &v1alpha1.Rollout{}
+			Expect(ReadYamlToObject("./test_data/custom/rollout.yaml", rollout)).ToNot(HaveOccurred())
+			CreateObject(rollout)
+
+			By("changing app version from v1 -> v2")
+			Expect(GetObject(app.Name, app)).NotTo(HaveOccurred())
+			app.Spec.Template.Spec.Volumes[0].ConfigMap.Name = "nginx-configmap2"
+			UpdateDeployment(app)
+			WaitRolloutCanaryStepPaused(rollout.Name, 1)
+			Expect(GetObject(virtualService.GetName(), virtualService)).ToNot(HaveOccurred())
+			Expect(ReadYamlToObject("./test_data/custom/expectVirtualService.yaml", expectVirtualService)).ToNot(HaveOccurred())
+			Expect(reflect.DeepEqual(virtualService.Object["spec"], expectVirtualService.Object["spec"])).To(BeTrue())
+
+			By("resume rollout")
+			ResumeRolloutCanary(rollout.Name)
+			WaitRolloutCanaryStepPaused(rollout.Name, 2)
+			Expect(GetObject(virtualService.GetName(), virtualService)).ToNot(HaveOccurred())
+			Expect(ReadYamlToObject("./test_data/custom/expectVirtualService2.yaml", expectVirtualService)).ToNot(HaveOccurred())
+			Expect(reflect.DeepEqual(virtualService.Object["spec"], expectVirtualService.Object["spec"])).To(BeTrue())
+		})
+	})
 })
 
 func mergeEnvVar(original []v1.EnvVar, add v1.EnvVar) []v1.EnvVar {

test/e2e/test_data/custom/appv1.yaml

@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  namespace: demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80
+        volumeMounts:
+        - name: html-volume
+          mountPath: /usr/share/nginx/html
+      volumes:
+      - name: html-volume
+        configMap:
+          name: nginx-configmap1

test/e2e/test_data/custom/expectVirtualService.yaml

@@ -0,0 +1,27 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: nginx-vs
+  namespace: demo
+spec:
+  hosts:
+  - "*"
+  gateways:
+  - nginx-gateway
+  http:
+  - match:
+    - headers:
+        user-agent:
+          exact: pc
+        name:
+          regex: .*demo
+    route:
+    - destination:
+        host: nginx-service
+      weight: 80
+    - destination:
+        host: nginx-service-canary
+      weight: 20
+  - route:
+    - destination:
+        host: nginx-service

test/e2e/test_data/custom/expectVirtualService2.yaml

@@ -0,0 +1,18 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: nginx-vs
+  namespace: demo
+spec:
+  hosts:
+  - "*"
+  gateways:
+  - nginx-gateway
+  http:
+  - route:
+    - destination:
+        host: nginx-service
+      weight: 50
+    - destination:
+        host: nginx-service-canary
+      weight: 50

test/e2e/test_data/custom/index1.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-configmap1
+  namespace: demo
+data:
+  index.html: |
+    <h1>Hello from nginx-v1</h1>

test/e2e/test_data/custom/index2.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-configmap2
+  namespace: demo
+data:
+  index.html: |
+    <h1>Hello from nginx-v2</h1>

test/e2e/test_data/custom/rollout.yaml

@@ -0,0 +1,33 @@
+apiVersion: rollouts.kruise.io/v1alpha1
+kind: Rollout
+metadata:
+  name: rollouts-demo
+  namespace: demo
+  annotations:
+    rollouts.kruise.io/rolling-style: canary
+spec:
+  disabled: false
+  objectRef:
+    workloadRef:
+      apiVersion: apps/v1
+      kind: Deployment
+      name: nginx-deployment
+  strategy:
+    canary:
+      steps:
+      - weight: 20
+        matches:
+        - headers:
+          - type: Exact
+            name: user-agent
+            value: pc
+          - type: RegularExpression
+            name: name
+            value: ".*demo"
+      - weight: 50
+      trafficRoutings:
+      - service: nginx-service
+        networkRefs:
+        - apiVersion: networking.istio.io/v1alpha3
+          kind: VirtualService
+          name: nginx-vs

test/e2e/test_data/custom/service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx-service
+  namespace: demo
+spec:
+  selector:
+    app: nginx
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80
+      name: http

test/e2e/test_data/custom/virtualService.yaml

@@ -0,0 +1,14 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: nginx-vs
+  namespace: demo
+spec:
+  hosts:
+  - "*"
+  gateways:
+  - nginx-gateway
+  http:
+  - route:
+    - destination:
+        host: nginx-service