add e2e tests for custom network provider (#177)

Signed-off-by: Kuromesi <blackfacepan@163.com>

.github/workflows/e2e-custom.yaml

@@ -0,0 +1,85 @@
+name: E2E-Custom
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+env:
+  # Common versions
+  GO_VERSION: '1.17'
+  KIND_IMAGE: 'kindest/node:v1.23.3'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+
+  rollout:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@v1.2.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise Rollout
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "1" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-rollout -o yaml
+          set -e
+          if [ "$PODS" -eq "1" ]; then
+            echo "Wait for kruise-rollout ready successfully"
+          else
+            echo "Timeout to wait for kruise-rollout ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          kubectl apply -f ./test/e2e/test_data/customNetworkProvider/istio_crd.yaml
+          kubectl apply -f ./test/e2e/test_data/customNetworkProvider/lua_script_configmap.yaml
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='Canary rollout with custon network provider' test/e2e
+          retVal=$?
+          # kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout
+          restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-rollout has not restarted"
+          else
+              kubectl get pod -n kruise-rollout --no-headers
+              echo "Kruise-rollout has restarted, abort!!!"
+              kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
+              exit 1
+          fi
+          exit $retVal

config/rbac/role.yaml

@@ -248,6 +248,17 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - networking.istio.io
+  resources:
+  - destinationrules
+  - virtualservices
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - networking.k8s.io
   resources:

pkg/controller/rollout/rollout_controller.go

@@ -83,6 +83,7 @@ type RolloutReconciler struct {
 //+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch
 // +kubebuilder:rbac:groups=apps.kruise.io,resources=daemonsets,verbs=get;list;watch;update;patch
 // +kubebuilder:rbac:groups=apps.kruise.io,resources=daemonsets/status,verbs=get;update;patch
+//+kubebuilder:rbac:groups=networking.istio.io,resources=virtualservices;destinationrules,verbs=get;list;watch;update;patch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

rollouts.rollouts.kruise.io-rollouts-demo.yaml

@@ -0,0 +1,60 @@
+apiVersion: rollouts.kruise.io/v1alpha1
+kind: Rollout
+metadata:
+  annotations:
+    rollouts.kruise.io/hash: d8c8v2w74286d77c4925wdw58v65z8725z7z4cw4f6cc485cvv62vx9cdwfv7b76
+    rollouts.kruise.io/rolling-style: canary
+  creationTimestamp: "2023-09-25T13:49:00Z"
+  deletionGracePeriodSeconds: 0
+  deletionTimestamp: "2023-09-25T13:52:34Z"
+  generation: 2
+  name: rollouts-demo
+  namespace: rollout-59e5b9cbeb2ed91f
+  resourceVersion: "2699"
+  uid: e861a61c-07f6-4064-8486-154bc6d86e29
+spec:
+  disabled: false
+  objectRef:
+    workloadRef:
+      apiVersion: apps/v1
+      kind: Deployment
+      name: echoserver
+  strategy:
+    canary:
+      steps:
+      - matches:
+        - headers:
+          - name: user-agent
+            type: Exact
+            value: pc
+        pause: {}
+        replicas: 1
+      - pause: {}
+        weight: 50
+      trafficRoutings:
+      - customNetworkRefs:
+        - apiVersion: networking.istio.io/v1alpha3
+          kind: VirtualService
+          name: vs-demo
+        service: echoserver
+status:
+  canaryStatus:
+    canaryReadyReplicas: 0
+    canaryReplicas: 0
+    canaryRevision: 684554996d
+    currentStepIndex: 2
+    currentStepState: Completed
+    observedWorkloadGeneration: 1
+    podTemplateHash: ""
+    rolloutHash: d8c8v2w74286d77c4925wdw58v65z8725z7z4cw4f6cc485cvv62vx9cdwfv7b76
+    stableRevision: 567694c97d
+  conditions:
+  - lastTransitionTime: "2023-09-25T13:50:00Z"
+    lastUpdateTime: "2023-09-25T13:50:00Z"
+    message: Rollout is in Progressing
+    reason: Initializing
+    status: "True"
+    type: Progressing
+  message: workload deployment is completed
+  observedGeneration: 1
+  phase: Progressing

test/e2e/rollout_test.go

@@ -34,6 +34,7 @@ import (
 	netv1 "k8s.io/api/networking/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/util/retry"
@@ -45,6 +46,7 @@ import (
 
 const (
 	nginxIngressAnnotationDefaultPrefix = "nginx.ingress.kubernetes.io"
+	OriginalSpecAnnotation              = "rollouts.kruise.io/original-spec-configuration"
 )
 
 var _ = SIGDescribe("Rollout", func() {
@@ -2606,6 +2608,227 @@ var _ = SIGDescribe("Rollout", func() {
 		})
 	})
 
+	KruiseDescribe("Canary rollout with custon network provider", func() {
+		It("V1->V2: Route traffic with header matches and weight using rollout for VirtualService", func() {
+			By("Creating Rollout...")
+			rollout := &v1alpha1.Rollout{}
+			Expect(ReadYamlToObject("./test_data/customNetworkProvider/rollout_with_trafficrouting.yaml", rollout)).ToNot(HaveOccurred())
+			CreateObject(rollout)
+
+			By("Creating workload and waiting for all pods ready...")
+			// service
+			service := &v1.Service{}
+			Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred())
+			CreateObject(service)
+			// istio api
+			vs := &unstructured.Unstructured{}
+			Expect(ReadYamlToObject("./test_data/customNetworkProvider/virtualservice_without_destinationrule.yaml", vs)).ToNot(HaveOccurred())
+			vs.SetAPIVersion("networking.istio.io/v1alpha3")
+			vs.SetKind("VirtualService")
+			CreateObject(vs)
+			// workload
+			workload := &apps.Deployment{}
+			Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred())
+			workload.Spec.Replicas = utilpointer.Int32(4)
+			CreateObject(workload)
+			WaitDeploymentAllPodsReady(workload)
+
+			// check rollout status
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy))
+			By("check rollout status & paused success")
+
+			// v1 -> v2, start rollout action
+			newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"})
+			workload.Spec.Template.Spec.Containers[0].Env = newEnvs
+			UpdateDeployment(workload)
+			By("Update deployment env NODE_NAME from(version1) -> to(version2), routing traffic with header agent:pc to new version pods")
+			time.Sleep(time.Second * 2)
+
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			Expect(workload.Spec.Paused).Should(BeTrue())
+			// wait step 1 complete
+			WaitRolloutCanaryStepPaused(rollout.Name, 1)
+			// check rollout status
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1))
+			Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1))
+			// check virtualservice spec
+			Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
+			expectedSpec := `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"match":[{"headers":{"user-agent":{"exact":"pc"}}}],"route":[{"destination":{"host":"echoserver-canary"}}]},{"route":[{"destination":{"host":"echoserver"}}]}]}`
+			Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedSpec))
+			// check original spec annotation
+			expectedAnno := `{"spec":{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}}`
+			Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(expectedAnno))
+
+			// resume rollout canary
+			ResumeRolloutCanary(rollout.Name)
+			By("Resume rollout, and wait next step(2), routing 50% traffic to new version pods")
+			WaitRolloutCanaryStepPaused(rollout.Name, 2)
+			// check rollout status
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 2))
+			Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 2))
+			// check virtualservice spec
+			Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
+			expectedSpec = `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":50},{"destination":{"host":"echoserver-canary"},"weight":50}]}]}`
+			Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedSpec))
+			// check original spec annotation
+			Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(expectedAnno))
+
+			// resume rollout
+			ResumeRolloutCanary(rollout.Name)
+			WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy)
+			By("rollout completed, and check")
+			// check service & virtualservice & deployment
+			// virtualservice
+			Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
+			expectedSpec = `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`
+			Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedSpec))
+			Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(""))
+			// service
+			Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
+			Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(""))
+			cService := &v1.Service{}
+			Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred())
+			// deployment
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			Expect(workload.Spec.Paused).Should(BeFalse())
+			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
+			Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas))
+			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
+			for _, env := range workload.Spec.Template.Spec.Containers[0].Env {
+				if env.Name == "NODE_NAME" {
+					Expect(env.Value).Should(Equal("version2"))
+				}
+			}
+			// check progressing succeed
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing)
+			Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted))
+			Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse)))
+			cond = util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded)
+			Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue)))
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation)
+		})
+
+		It("V1->V2: Route traffic with header matches and weight for VirtualService and DestinationRule", func() {
+			By("Creating Rollout...")
+			rollout := &v1alpha1.Rollout{}
+			Expect(ReadYamlToObject("./test_data/customNetworkProvider/rollout_without_trafficrouting.yaml", rollout)).ToNot(HaveOccurred())
+			CreateObject(rollout)
+
+			By("Creating TrafficRouting...")
+			traffic := &v1alpha1.TrafficRouting{}
+			Expect(ReadYamlToObject("./test_data/customNetworkProvider/trafficrouting.yaml", traffic)).ToNot(HaveOccurred())
+			CreateObject(traffic)
+
+			By("Creating workload and waiting for all pods ready...")
+			// service
+			service := &v1.Service{}
+			Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred())
+			CreateObject(service)
+			// istio api
+			vs := &unstructured.Unstructured{}
+			Expect(ReadYamlToObject("./test_data/customNetworkProvider/virtualservice_without_destinationrule.yaml", vs)).ToNot(HaveOccurred())
+			vs.SetAPIVersion("networking.istio.io/v1alpha3")
+			vs.SetKind("VirtualService")
+			CreateObject(vs)
+			dr := &unstructured.Unstructured{}
+			Expect(ReadYamlToObject("./test_data/customNetworkProvider/destinationrule.yaml", dr)).ToNot(HaveOccurred())
+			dr.SetAPIVersion("networking.istio.io/v1alpha3")
+			dr.SetKind("DestinationRule")
+			CreateObject(dr)
+			// workload
+			workload := &apps.Deployment{}
+			Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred())
+			workload.Spec.Replicas = utilpointer.Int32(4)
+			CreateObject(workload)
+			WaitDeploymentAllPodsReady(workload)
+
+			// check rollout and trafficrouting status
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			Expect(rollout.Status.Phase).Should(Equal(v1alpha1.RolloutPhaseHealthy))
+			Expect(GetObject(traffic.Name, traffic)).NotTo(HaveOccurred())
+			Expect(traffic.Status.Phase).Should(Equal(v1alpha1.TrafficRoutingPhaseHealthy))
+			By("check rollout and trafficrouting status & paused success")
+
+			// v1 -> v2, start rollout action
+			newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"})
+			workload.Spec.Template.Spec.Containers[0].Env = newEnvs
+			UpdateDeployment(workload)
+			By("Update deployment env NODE_NAME from(version1) -> to(version2), routing traffic with header agent:pc to new version pods")
+			time.Sleep(time.Second * 2)
+
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			Expect(workload.Spec.Paused).Should(BeTrue())
+			// wait step 1 complete
+			WaitRolloutCanaryStepPaused(rollout.Name, 1)
+			// check rollout and trafficrouting status
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1))
+			Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1))
+			Expect(GetObject(traffic.Name, traffic)).NotTo(HaveOccurred())
+			Expect(traffic.Status.Phase).Should(Equal(v1alpha1.TrafficRoutingPhaseProgressing))
+			// check virtualservice and destinationrule spec
+			Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
+			expectedVSSpec := `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"match":[{"headers":{"user-agent":{"exact":"pc"}}}],"route":[{"destination":{"host":"echoserver","subset":"canary"}}]},{"route":[{"destination":{"host":"echoserver"}}]}]}`
+			Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedVSSpec))
+			Expect(GetObject(dr.GetName(), dr)).NotTo(HaveOccurred())
+			expectedDRSpec := `{"host":"svc-demo","subsets":[{"labels":{"version":"base"},"name":"echoserver"},{"labels":{"istio.service.tag":"gray"},"name":"canary"}],"trafficPolicy":{"loadBalancer":{"simple":"ROUND_ROBIN"}}}`
+			Expect(util.DumpJSON(dr.Object["spec"])).Should(Equal(expectedDRSpec))
+			// check original spec annotation
+			expectedVSAnno := `{"spec":{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}}`
+			Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(expectedVSAnno))
+			expectedDRAnno := `{"spec":{"host":"svc-demo","subsets":[{"labels":{"version":"base"},"name":"echoserver"}],"trafficPolicy":{"loadBalancer":{"simple":"ROUND_ROBIN"}}}}`
+			Expect(dr.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(expectedDRAnno))
+
+			// resume rollout
+			ResumeRolloutCanary(rollout.Name)
+			WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy)
+			Expect(GetObject(traffic.Name, traffic)).NotTo(HaveOccurred())
+			Expect(traffic.Status.Phase).Should(Equal(v1alpha1.TrafficRoutingPhaseHealthy))
+			By("rollout completed, and check")
+			// check service & virtualservice & destinationrule & deployment
+			// virtualservice and destinationrule
+			Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
+			expectedVSSpec = `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`
+			Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedVSSpec))
+			Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(""))
+
+			Expect(GetObject(dr.GetName(), dr)).NotTo(HaveOccurred())
+			expectedDRSpec = `{"host":"svc-demo","subsets":[{"labels":{"version":"base"},"name":"echoserver"}],"trafficPolicy":{"loadBalancer":{"simple":"ROUND_ROBIN"}}}`
+			Expect(util.DumpJSON(dr.Object["spec"])).Should(Equal(expectedDRSpec))
+			Expect(dr.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(""))
+			// service
+			Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
+			Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(""))
+			cService := &v1.Service{}
+			Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred())
+			// deployment
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			Expect(workload.Spec.Paused).Should(BeFalse())
+			Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
+			Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas))
+			Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
+			for _, env := range workload.Spec.Template.Spec.Containers[0].Env {
+				if env.Name == "NODE_NAME" {
+					Expect(env.Value).Should(Equal("version2"))
+				}
+			}
+			// check progressing succeed
+			Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
+			cond := util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionProgressing)
+			Expect(cond.Reason).Should(Equal(v1alpha1.ProgressingReasonCompleted))
+			Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse)))
+			cond = util.GetRolloutCondition(rollout.Status, v1alpha1.RolloutConditionSucceeded)
+			Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue)))
+			Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
+			WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation)
+		})
+	})
+
 	KruiseDescribe("CloneSet canary rollout with Ingress", func() {
 		It("CloneSet V1->V2: Percentage, 20%,60% Succeeded", func() {
 			By("Creating Rollout...")

test/e2e/test_data/customNetworkProvider/destinationrule.yaml

@@ -0,0 +1,13 @@
+apiVersion: networking.istio.io/v1beta1
+kind: DestinationRule
+metadata:
+  name: ds-demo
+spec:
+  host: svc-demo
+  trafficPolicy:
+    loadBalancer:
+      simple: ROUND_ROBIN
+  subsets:
+    - labels:
+        version: base
+      name: echoserver

test/e2e/test_data/customNetworkProvider/lua_script_configmap.yaml

@@ -0,0 +1,181 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kruise-rollout-configuration
+  namespace: kruise-rollout
+data:
+  lua.traffic.routing.VirtualService.networking.istio.io: |
+    spec = obj.data.spec
+
+    if obj.canaryWeight == -1 then
+        obj.canaryWeight = 100
+        obj.stableWeight = 0
+    end
+
+    function FindAllRules(spec, protocol)
+        local rules = {}
+        if (spec[protocol] ~= nil) then
+            for _, proto in ipairs(spec[protocol]) do
+                table.insert(rules, proto)
+            end
+        end
+        return rules
+    end
+
+    -- find matched route of VirtualService spec with stable svc
+    function FindMatchedRules(spec, stableService, protocol)
+        local matchedRoutes = {}
+        local rules = FindAllRules(spec, protocol)
+        -- a rule contains 'match' and 'route'
+        for _, rule in ipairs(rules) do
+            -- skip routes with matches
+            if (rule.match == nil) then
+                for _, route in ipairs(rule.route) do
+                    if route.destination.host == stableService then
+                        table.insert(matchedRoutes, rule)
+                        break
+                    end
+                end
+            end
+        end
+        return matchedRoutes
+    end
+
+    function HasMatchedRule(spec, stableService, protocol)
+        local rules = FindAllRules(spec, protocol)
+        -- a rule contains 'match' and 'route'
+        for _, rule in ipairs(rules) do
+            for _, route in ipairs(rule.route) do
+                if route.destination.host == stableService then
+                    return true
+                end
+            end
+        end
+        return false
+    end
+
+    function DeepCopy(original)
+        local copy
+        if type(original) == 'table' then
+            copy = {}
+            for key, value in pairs(original) do
+                copy[key] = DeepCopy(value)
+            end
+        else
+            copy = original
+        end
+        return copy
+    end
+
+    function CalculateWeight(route, stableWeight, n)
+        local weight
+        if (route.weight) then
+            weight = math.floor(route.weight * stableWeight / 100)
+        else
+            weight = math.floor(stableWeight / n)
+        end
+        return weight
+    end
+
+    -- generate routes with matches, insert a rule before other rules
+    function GenerateMatchedRoutes(spec, matches, stableService, canaryService, stableWeight, canaryWeight, protocol)
+        for _, match in ipairs(matches) do
+            local route = {}
+            route["match"] = {}
+            if (not HasMatchedRule(spec, stableService, protocol)) then
+                return
+            end
+            for key, value in pairs(match) do
+                local vsMatch = {}
+                vsMatch[key] = {}
+                for _, rule in ipairs(value) do
+                    if rule["type"] == "RegularExpression" then
+                        matchType = "regex"
+                    elseif rule["type"] == "Exact" then
+                        matchType = "exact"
+                    elseif rule["type"] == "Prefix" then
+                        matchType = "prefix"
+                    end
+                    if key == "headers" then
+                        vsMatch[key][rule["name"]] = {}
+                        vsMatch[key][rule["name"]][matchType] = rule.value
+                    else
+                        vsMatch[key][matchType] = rule.value
+                    end
+                end
+                table.insert(route["match"], vsMatch)
+            end
+            route.route = {
+                {
+                    destination = {}
+                }
+            }
+            -- if stableService == canaryService, then do e2e release
+            if stableService == canaryService then
+                route.route[1].destination.host = stableService
+                route.route[1].destination.subset = "canary"
+            else
+                route.route[1].destination.host = canaryService
+            end
+            if (protocol == "http") then
+                table.insert(spec.http, 1, route)
+            elseif (protocol == "tls") then
+                table.insert(spec.tls, 1, route)
+            elseif (protocol == "tcp") then
+                table.insert(spec.tcp, 1, route)
+            end
+        end
+    end
+
+    -- generate routes without matches, change every rule
+    function GenerateRoutes(spec, stableService, canaryService, stableWeight, canaryWeight, protocol)
+        local matchedRules = FindMatchedRules(spec, stableService, protocol)
+        for _, rule in ipairs(matchedRules) do
+            local canary
+            if stableService ~= canaryService then
+                canary = {
+                    destination = {
+                        host = canaryService,
+                    },
+                    weight = canaryWeight,
+                }
+            else
+                canary = {
+                    destination = {
+                        host = stableService,
+                        subset = "canary",
+                    },
+                    weight = canaryWeight,
+                }
+            end
+
+            -- incase there are multiple versions traffic already, do a for-loop
+            for _, route in ipairs(rule.route) do
+                -- update stable service weight
+                route.weight = CalculateWeight(route, stableWeight, #rule.route)
+            end
+            table.insert(rule.route, canary)
+        end
+    end
+
+    if (obj.matches)
+    then
+        GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "http")
+        GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tcp")
+        GenerateMatchedRoutes(spec, obj.matches, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tls")
+    else
+        GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "http")
+        GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tcp")
+        GenerateRoutes(spec, obj.stableService, obj.canaryService, obj.stableWeight, obj.canaryWeight, "tls")
+    end
+    return obj.data
+
+  lua.traffic.routing.DestinationRule.networking.istio.io: |
+    local spec = obj.data.spec
+    local canary = {}
+    canary.labels = {}
+    canary.name = "canary"
+    local podLabelKey = "istio.service.tag"
+    canary.labels[podLabelKey] = "gray"
+    table.insert(spec.subsets, canary)
+    return obj.data

test/e2e/test_data/customNetworkProvider/rollout_with_trafficrouting.yaml

@@ -0,0 +1,29 @@
+  apiVersion: rollouts.kruise.io/v1alpha1
+  kind: Rollout
+  metadata:
+    name: rollouts-demo
+    annotations:
+      rollouts.kruise.io/rolling-style: canary
+  spec:
+    disabled: false
+    objectRef:
+      workloadRef:
+        apiVersion: apps/v1
+        kind: Deployment
+        name: echoserver
+    strategy:
+      canary:
+        steps:
+        - replicas: 1
+          matches:
+          - headers:
+            - type: Exact
+              name: user-agent
+              value: pc
+        - weight: 50
+        trafficRoutings:
+        - service: echoserver
+          customNetworkRefs:
+          - apiVersion: networking.istio.io/v1alpha3
+            kind: VirtualService
+            name: vs-demo

test/e2e/test_data/customNetworkProvider/rollout_without_trafficrouting.yaml

@@ -0,0 +1,18 @@
+  apiVersion: rollouts.kruise.io/v1alpha1
+  kind: Rollout
+  metadata:
+    name: rollouts-demo
+    annotations:
+      rollouts.kruise.io/rolling-style: canary
+      rollouts.kruise.io/trafficrouting: tr-demo
+  spec:
+    disabled: false
+    objectRef:
+      workloadRef:
+        apiVersion: apps/v1
+        kind: Deployment
+        name: echoserver
+    strategy:
+      canary:
+        steps:
+        - replicas: 1

test/e2e/test_data/customNetworkProvider/trafficrouting.yaml

@@ -0,0 +1,20 @@
+apiVersion: rollouts.kruise.io/v1alpha1
+kind: TrafficRouting
+metadata:
+  name: tr-demo
+spec:
+  strategy:
+    matches:
+    - headers:
+      - type: Exact
+        name: user-agent
+        value: pc
+  objectRef:
+  - service: echoserver
+    customNetworkRefs:
+    - apiVersion: networking.istio.io/v1alpha3
+      kind: VirtualService
+      name: vs-demo
+    - apiVersion: networking.istio.io/v1alpha3
+      kind: DestinationRule
+      name: ds-demo

test/e2e/test_data/customNetworkProvider/virtualservice_without_destinationrule.yaml

@@ -0,0 +1,13 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: vs-demo
+spec:
+  hosts:
+  - "*"
+  gateways:
+  - nginx-gateway
+  http:
+  - route:
+    - destination:
+        host: echoserver