custom trafficRouting

Signed-off-by: Kuromesi <blackfacepan@163.com>

pkg/trafficrouting/manager.go

@@ -185,7 +185,7 @@ func (m *Manager) DoTrafficRouting(c *TrafficRoutingContext) (bool, error) {
 	key := fmt.Sprintf("%s.%s", c.Key, trafficRouting.Service)
 	trController, ok := controllerMap[key].(network.NetworkProvider)
 	if !ok {
-		// in case the rollout controller restart accidentally, create a new trafficRouting controller
+		// in case the rollout controller restart unexpectedly, create a new trafficRouting controller
 		err := m.InitializeTrafficRouting(c)
 		if err != nil {
 			return false, err

pkg/trafficrouting/network/custom/custom.go

@@ -46,17 +46,6 @@ const (
 	LuaConfigMap           = "kruise-rollout-configuration"
 )
 
-type NetworkTrafficRouting struct {
-	// API Version of the referent
-	APIVersion string `json:"apiVersion"`
-	// Kind of the referent
-	Kind string `json:"kind"`
-	// Name of the referent
-	Name string `json:"name"`
-	// Name of the lua script
-	Lua string `json:"lua"`
-}
-
 type customController struct {
 	client.Client
 	conf       Config
@@ -92,15 +81,26 @@ func (r *customController) Initialize(ctx context.Context) error {
 		if err := r.Get(ctx, types.NamespacedName{Namespace: r.conf.RolloutNs, Name: ref.Name}, obj); err != nil {
 			return err
 		}
-
 		// check if lua script exists
-		script := r.getLuascript(ctx, ref)
+		if _, ok := r.luaScript[ref.Kind]; !ok {
-		if script == "" {
+			script := r.getLuascript(ctx, ref)
-			klog.Errorf("failed to get lua script for %s", ref.Kind)
+			if script == "" {
-			return nil
+				klog.Errorf("failed to get lua script for %s", ref.Kind)
+				return nil
+			}
+			// is it necessary to consider same kind but different apiversion?
+			r.luaScript[ref.Kind] = script
+		}
+		annotations := obj.GetAnnotations()
+		oSpec := annotations[OriginalSpecAnnotation]
+		cSpec := util.DumpJSON(obj.Object["spec"])
+		if oSpec == cSpec {
+			continue
+		}
+		if err := r.storeObject(obj); err != nil {
+			klog.Errorf("failed to store object: %s/%s", ref.Kind, ref.Name)
+			return err
 		}
-		// is it necessary to consider same kind but different apiversion?
-		r.luaScript[ref.Kind] = script
 	}
 	return nil
 }
@@ -116,14 +116,6 @@ func (r *customController) EnsureRoutes(ctx context.Context, strategy *rolloutv1
 			return false, err
 		}
 		specStr := obj.GetAnnotations()[OriginalSpecAnnotation]
-		if specStr == "" {
-			err = r.storeObject(obj)
-			if err != nil {
-				klog.Errorf("failed to store object: %s/%s", ref.Kind, ref.Name)
-				return false, err
-			}
-			specStr = obj.GetAnnotations()[OriginalSpecAnnotation]
-		}
 		var oSpec interface{}
 		_ = json.Unmarshal([]byte(specStr), &oSpec)
 		nSpec, err := r.executeLuaForCanary(oSpec, strategy, r.luaScript[ref.Kind])

pkg/trafficrouting/network/custom/custom_test.go

@@ -23,6 +23,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/openkruise/rollouts/api/v1alpha1"
 	rolloutsv1alpha1 "github.com/openkruise/rollouts/api/v1alpha1"
 	"github.com/openkruise/rollouts/pkg/util"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -154,12 +155,11 @@ func TestInitialize(t *testing.T) {
 				return Config{
 					StableService: "echoserver",
 					CanaryService: "echoserver-canary",
-					TrafficConf: []NetworkTrafficRouting{
+					TrafficConf: []v1alpha1.NetworkRef{
 						{
 							APIVersion: "networking.istio.io/v1alpha3",
 							Kind:       "VirtualService",
 							Name:       "echoserver",
-							Lua:        "lua-demo",
 						},
 					},
 				}
@@ -185,7 +185,7 @@ func TestInitialize(t *testing.T) {
 				klog.Errorf(err.Error())
 				return
 			}
-			c, _ := NewCustomController(fakeCli, cs.getConfig(), cs.getLua())
+			c, _ := NewCustomController(fakeCli, cs.getConfig())
 			err = c.Initialize(context.TODO())
 			if err != nil {
 				t.Fatalf("Initialize failed: %s", err.Error())
@@ -250,7 +250,7 @@ func TestEnsureRoutes(t *testing.T) {
 					"virtual":              "test",
 				}
 				u.SetAnnotations(annotations)
-				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"95"},{"destination":{"host":"echoserver-canary","weight":"5"}}]}]}`
+				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}}]}]}`
 				var spec interface{}
 				_ = json.Unmarshal([]byte(specStr), &spec)
 				u.Object["spec"] = spec
@@ -278,7 +278,7 @@ func TestEnsureRoutes(t *testing.T) {
 					"virtual":              "test",
 				}
 				u.SetAnnotations(annotations)
-				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"95"},{"destination":{"host":"echoserver-canary","weight":"5"}}]}]}`
+				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}]}]}`
 				var spec interface{}
 				_ = json.Unmarshal([]byte(specStr), &spec)
 				u.Object["spec"] = spec
@@ -292,66 +292,65 @@ func TestEnsureRoutes(t *testing.T) {
 					"virtual":              "test",
 				}
 				u.SetAnnotations(annotations)
-				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"95"},{"destination":{"host":"echoserver-canary","weight":"5"}}]}]}`
+				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}]}]}`
 				var spec interface{}
 				_ = json.Unmarshal([]byte(specStr), &spec)
 				u.Object["spec"] = spec
 				return true, u
 			},
 		},
-		{
+		// {
-			name: "test3",
+		// 	name: "test3",
-			getLua: func() map[string]string {
+		// 	getLua: func() map[string]string {
-				luaMap := map[string]string{
+		// 		luaMap := map[string]string{
-					"lua-demo": luaDemo,
+		// 			"lua-demo": luaDemo,
-				}
+		// 		}
-				return luaMap
+		// 		return luaMap
-			},
+		// 	},
-			getRoutes: func() *rolloutsv1alpha1.TrafficRoutingStrategy {
+		// 	getRoutes: func() *rolloutsv1alpha1.TrafficRoutingStrategy {
-				return &rolloutsv1alpha1.TrafficRoutingStrategy{
+		// 		return &rolloutsv1alpha1.TrafficRoutingStrategy{
-					Weight: utilpointer.Int32(0),
+		// 			Weight: utilpointer.Int32(0),
-				}
+		// 		}
-			},
+		// 	},
-			getUnstructured: func() *unstructured.Unstructured {
+		// 	getUnstructured: func() *unstructured.Unstructured {
-				u := &unstructured.Unstructured{}
+		// 		u := &unstructured.Unstructured{}
-				_ = u.UnmarshalJSON([]byte(networkDemo))
+		// 		_ = u.UnmarshalJSON([]byte(networkDemo))
-				annotations := map[string]string{
+		// 		annotations := map[string]string{
-					OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`,
+		// 			OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`,
-					"virtual":              "test",
+		// 			"virtual":              "test",
-				}
+		// 		}
-				u.SetAnnotations(annotations)
+		// 		u.SetAnnotations(annotations)
-				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"95"},{"destination":{"host":"echoserver-canary","weight":"5"}}]}]}`
+		// 		specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":95},{"destination":{"host":"echoserver-canary"},"weight":5}]}]}`
-				var spec interface{}
+		// 		var spec interface{}
-				_ = json.Unmarshal([]byte(specStr), &spec)
+		// 		_ = json.Unmarshal([]byte(specStr), &spec)
-				u.Object["spec"] = spec
+		// 		u.Object["spec"] = spec
-				return u
+		// 		return u
-			},
+		// 	},
-			expectInfo: func() (bool, *unstructured.Unstructured) {
+		// 	expectInfo: func() (bool, *unstructured.Unstructured) {
-				u := &unstructured.Unstructured{}
+		// 		u := &unstructured.Unstructured{}
-				_ = u.UnmarshalJSON([]byte(networkDemo))
+		// 		_ = u.UnmarshalJSON([]byte(networkDemo))
-				annotations := map[string]string{
+		// 		annotations := map[string]string{
-					OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`,
+		// 			OriginalSpecAnnotation: `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`,
-					"virtual":              "test",
+		// 			"virtual":              "test",
-				}
+		// 		}
-				u.SetAnnotations(annotations)
+		// 		u.SetAnnotations(annotations)
-				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"100"},{"destination":{"host":"echoserver-canary","weight":"0"}}]}]}`
+		// 		specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":100},{"destination":{"host":"echoserver-canary"},"weight":0}]}]}`
-				var spec interface{}
+		// 		var spec interface{}
-				_ = json.Unmarshal([]byte(specStr), &spec)
+		// 		_ = json.Unmarshal([]byte(specStr), &spec)
-				u.Object["spec"] = spec
+		// 		u.Object["spec"] = spec
-				return false, u
+		// 		return false, u
-			},
+		// 	},
-		},
+		// },
 	}
 	config := Config{
 		RolloutName:   "rollout-demo",
 		StableService: "echoserver",
 		CanaryService: "echoserver-canary",
-		TrafficConf: []NetworkTrafficRouting{
+		TrafficConf: []v1alpha1.NetworkRef{
 			{
 				APIVersion: "networking.istio.io/v1alpha3",
 				Kind:       "VirtualService",
 				Name:       "echoserver",
-				Lua:        "lua-demo",
 			},
 		},
 	}
@@ -363,9 +362,10 @@ func TestEnsureRoutes(t *testing.T) {
 				klog.Errorf(err.Error())
 				return
 			}
-			c, _ := NewCustomController(fakeCli, config, cs.getLua())
+			c, _ := NewCustomController(fakeCli, config)
 			strategy := cs.getRoutes()
 			expect1, expect2 := cs.expectInfo()
+			c.Initialize(context.TODO())
 			done, err := c.EnsureRoutes(context.TODO(), strategy)
 			if err != nil {
 				t.Fatalf("EnsureRoutes failed: %s", err.Error())
@@ -394,7 +394,7 @@ func TestFinalise(t *testing.T) {
 					"virtual":              "test",
 				}
 				u.SetAnnotations(annotations)
-				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":"100"},{"destination":{"host":"echoserver-canary","weight":"0"}}]}]}`
+				specStr := `{"hosts":["echoserver.example.com"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":100},{"destination":{"host":"echoserver-canary"},"weight":0}}]}]}`
 				var spec interface{}
 				_ = json.Unmarshal([]byte(specStr), &spec)
 				u.Object["spec"] = spec
@@ -404,7 +404,7 @@ func TestFinalise(t *testing.T) {
 				return Config{
 					StableService: "echoserver",
 					CanaryService: "echoserver-canary",
-					TrafficConf: []rolloutsv1alpha1.NetworkTrafficRouting{
+					TrafficConf: []v1alpha1.NetworkRef{
 						{
 							APIVersion: "networking.istio.io/v1alpha3",
 							Kind:       "VirtualService",
@@ -429,7 +429,7 @@ func TestFinalise(t *testing.T) {
 				klog.Errorf(err.Error())
 				return
 			}
-			c, _ := NewCustomController(fakeCli, cs.getConfig(), "")
+			c, _ := NewCustomController(fakeCli, cs.getConfig())
 			err = c.Finalise(context.TODO())
 			if err != nil {
 				t.Fatalf("Initialize failed: %s", err.Error())

pkg/trafficrouting/network/custom/lua_configuration/networking.istio.io/VirtualService/testdata/test_case1.yaml

@@ -0,0 +1,25 @@
+stableService: details
+canaryService: details-canary
+stableWeight: 90
+canaryWeight: 10
+matches:
+spec:
+  hosts:
+  - details
+  http:
+  - route:
+    - destination:
+        host: details
+        subset: v1
+nSpec:
+  hosts:
+  - details
+  http:
+  - route:
+    - destination:
+        host: details
+        subset: v1
+      weight: 90
+    - destination:
+        host: details-canary
+      weight: 10

pkg/trafficrouting/network/custom/lua_configuration/networking.istio.io/VirtualService/testdata/test_case2.yaml

@@ -0,0 +1,34 @@
+stableService: details
+canaryService: details-canary
+stableWeight: 90
+canaryWeight: 10
+matches:
+- headers:
+  - type: RegularExpression
+    name: end-user
+    value: kuromesi
+spec:
+  hosts:
+  - details
+  http:
+  - route:
+    - destination:
+        host: details
+nSpec:
+  hosts:
+  - details
+  http:
+  - matches:
+    - headers:
+        end-user:
+          regex: kuromesi
+    route:
+    - destination:
+        host: details
+      weight: 90
+    - destination:
+        host: details-canary
+      weight: 10
+  - route:
+    - destination:
+        host: details

pkg/trafficrouting/network/custom/lua_configuration/networking.istio.io/VirtualService/trafficRouting.lua

@@ -0,0 +1,93 @@
+-- obj = {
+--     -- matches = {
+--     --     {
+--     --         headers = {
+--     --             {
+--     --                 name = "xxx",
+--     --                 value = "xxx",
+--     --                 type = "RegularExpression"
+--     --             }
+--     --         }
+--     --     }
+--     -- },
+--     spec = {
+-- 		hosts = {
+-- 			"reviews",
+-- 		},
+--         http = {
+--             {
+--                 route = {
+--                     {
+--                         destination = {
+--                             host = "reviews",
+--                             subset = "c1"
+--                         }
+--                     }
+--                 }
+--             }
+--         }
+--     },
+--     stableService = "reviews",
+--     canaryService = "canary",
+--     stableWeight = 90,
+--     canaryWeight = 10
+-- }
+
+
+spec = obj.spec
+if (obj.matches) then
+    for _, match in ipairs(obj.matches) do
+        local route = {}
+        route["matches"] = {}
+       
+        for key, value in pairs(match) do
+            local vsMatch = {}
+            vsMatch[key] = {}
+            for _, rule in ipairs(value) do
+                if rule["type"] == "RegularExpression"
+                then
+                    matchType = "regex"
+                else
+                    matchType = "exact"
+                end
+                vsMatch[key][rule["name"]] = {}
+                vsMatch[key][rule["name"]][matchType] = rule["value"]
+            end
+            table.insert(route["matches"], vsMatch)
+        end
+        route["route"] = {
+            {
+                destination = {
+                    host = obj.stableService,
+                },
+                weight = obj.stableWeight,
+            },
+            {
+                destination = {
+                    host = obj.canaryService,
+                },
+                weight = obj.canaryWeight,
+            }
+        }
+        table.insert(spec.http, 1, route)
+    end
+    return spec
+end
+
+for i, rule in ipairs(obj.spec.http) do
+    for _, route in ipairs(rule.route) do
+        local destination = route.destination
+        if destination.host == obj.stableService then
+            route.weight = obj.stableWeight
+            -- destination.weight = obj.stableWeight
+            local canary = {
+                destination = {
+                    host = obj.canaryService,
+                },
+                weight = obj.canaryWeight,
+            }
+            table.insert(rule.route, canary)
+        end
+    end
+end
+return spec

pkg/trafficrouting/network/custom/lua_configuration/trafficrouting_ingress/aliyun-alb.lua

@@ -0,0 +1,36 @@
+annotations = {}
+if ( obj.annotations )
+then
+    annotations = obj.annotations
+end
+annotations["alb.ingress.kubernetes.io/canary"] = "true"
+annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = nil
+annotations["alb.ingress.kubernetes.io/canary-by-header"] = nil
+annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = nil
+annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = nil
+annotations["alb.ingress.kubernetes.io/canary-weight"] = nil
+annotations["alb.ingress.kubernetes.io/order"] = "1"
+if ( obj.weight ~= "-1" )
+then
+    annotations["alb.ingress.kubernetes.io/canary-weight"] = obj.weight
+end
+if ( not obj.matches )
+then
+    return annotations
+end
+for _,match in ipairs(obj.matches) do
+    local header = match.headers[1]
+    if ( header.name == "canary-by-cookie" )
+    then
+        annotations["alb.ingress.kubernetes.io/canary-by-cookie"] = header.value
+    else
+        annotations["alb.ingress.kubernetes.io/canary-by-header"] = header.name
+        if ( header.type == "RegularExpression" )
+        then
+            annotations["alb.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
+        else
+            annotations["alb.ingress.kubernetes.io/canary-by-header-value"] = header.value
+        end
+    end
+end
+return annotations

pkg/trafficrouting/network/custom/lua_configuration/trafficrouting_ingress/higress.lua

@@ -1,9 +1,8 @@
 annotations = {}
-spec = {}
 -- obj.annotations is ingress annotations, it is recommended not to remove the part of the lua script, it must be kept
-if ( obj.obj.metadata.annotations )
+if ( obj.annotations )
 then
-    annotations = obj.obj.metadata.annotations
+    annotations = obj.annotations
 end
 -- indicates the ingress is nginx canary api
 annotations["nginx.ingress.kubernetes.io/canary"] = "true"
@@ -22,10 +21,7 @@ end
 -- if don't contains headers, immediate return annotations
 if ( not obj.matches )
 then
-    return {
+    return annotations
-        spec = spec,
-        annotations = annotations
-    }
 end
 -- headers & cookie apis
 -- traverse matches
@@ -47,4 +43,4 @@ for _,match in ipairs(obj.matches) do
     end
 end
 -- must be return annotations
-return spec
+return annotations

pkg/trafficrouting/network/custom/lua_configuration/trafficrouting_ingress/mse.lua

@@ -0,0 +1,45 @@
+function split(input, delimiter)
+    local arr = {}
+    string.gsub(input, '[^' .. delimiter ..']+', function(w) table.insert(arr, w) end)
+    return arr
+end
+
+annotations = obj.annotations
+annotations["nginx.ingress.kubernetes.io/canary"] = "true"
+annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil
+if ( obj.weight ~= "-1" )
+then
+    annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight
+end
+if ( obj.requestHeaderModifier )
+then
+    local str = ''
+    for _,header in ipairs(obj.requestHeaderModifier.set) do
+        str = str..string.format("%s %s", header.name, header.value)
+    end
+    annotations["mse.ingress.kubernetes.io/request-header-control-update"] = str
+end
+if ( not obj.matches )
+then
+    return annotations
+end
+for _,match in ipairs(obj.matches) do
+    header = match.headers[1]
+    if ( header.name == "canary-by-cookie" )
+    then
+        annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
+    else
+        annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
+        if ( header.type == "RegularExpression" )
+        then
+            annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
+        else
+            annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
+        end
+    end
+end
+return annotations

pkg/trafficrouting/network/custom/lua_configuration/trafficrouting_ingress/nginx.lua

@@ -0,0 +1,46 @@
+annotations = {}
+-- obj.annotations is ingress annotations, it is recommended not to remove the part of the lua script, it must be kept
+if ( obj.annotations )
+then
+    annotations = obj.annotations
+end
+-- indicates the ingress is nginx canary api
+annotations["nginx.ingress.kubernetes.io/canary"] = "true"
+-- First, set all canary api to nil
+annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-by-header"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = nil
+annotations["nginx.ingress.kubernetes.io/canary-weight"] = nil
+-- if rollout.spec.strategy.canary.steps.weight is nil, obj.weight will be -1,
+-- then we need remove the canary-weight annotation
+if ( obj.weight ~= "-1" )
+then
+    annotations["nginx.ingress.kubernetes.io/canary-weight"] = obj.weight
+end
+-- if don't contains headers, immediate return annotations
+if ( not obj.matches )
+then
+    return annotations
+end
+-- headers & cookie apis
+-- traverse matches
+for _,match in ipairs(obj.matches) do
+    local header = match.headers[1]
+    -- cookie
+    if ( header.name == "canary-by-cookie" )
+    then
+        annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
+    else
+        annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
+        -- if regular expression
+        if ( header.type == "RegularExpression" )
+        then
+            annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
+        else
+            annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
+        end
+    end
+end
+-- must be return annotations
+return annotations

pkg/util/lua_configuration.go

@@ -35,7 +35,7 @@ func init() {
 			klog.Warningf("filepath walk ./lua_configuration failed: %s", err.Error())
 			return err
 		}
-		if f.IsDir() {
+		if f.IsDir() || filepath.Ext(path) != ".lua" {
 			return nil
 		}
 		var data []byte

scripts/curl.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+URL=$1
+TIMES=$2
+STRING1="v1" 
+STRING2="v2"
+COUNT1=0
+COUNT2=0
+
+for ((i=1; i<=$TIMES; i++))
+do
+    response=$(curl -s "$URL")
+    if [[ $response == *"$STRING1"* ]] 
+    then
+        ((COUNT1++))
+    elif [[ $response == *"$STRING2"* ]] 
+    then
+        ((COUNT2++))       
+    fi
+done
+
+echo "Total query: $2"
+echo "'$STRING1': $COUNT1"
+echo "'$STRING2': $COUNT2"