add e2e tests and update config/operator to suport e2e

Signed-off-by: Elieser Pereira <elieser.pereiraa@gmail.com>

Makefile

@@ -221,6 +221,9 @@ deploy: manifests kustomize ## Deploy to the K8s cluster specified in ~/.kube/co
 	cd config/operator && \
 	$(KUSTOMIZE) edit set image ghcr.io/kedacore/http-add-on-operator=${IMAGE_OPERATOR_VERSIONED_TAG}
 
+	cd config/operator && \
+	$(KUSTOMIZE) edit add patch --path e2e-test/otel/deployment.yaml --group apps --kind Deployment --name operator --version v1
+
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
 
 undeploy:

config/operator/deployment.yaml

@@ -38,7 +38,7 @@ spec:
           value: ""
         ports:
         - name: metrics
-          containerPort: 8080
+          containerPort: 2223
         - name: probes
           containerPort: 8081
         livenessProbe:

config/operator/e2e-test/otel/deployment.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: operator
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: operator
+        env:
+        - name: OTEL_PROM_EXPORTER_ENABLED
+          value: "true"
+        - name: OTEL_PROM_EXPORTER_PORT
+          value: "2223"
+        - name: OTEL_EXPORTER_OTLP_METRICS_ENABLED
+          value: "true"
+        - name: OTEL_EXPORTER_OTLP_ENDPOINT
+          value: "http://opentelemetry-collector.open-telemetry-system:4318"
+        - name: OTEL_METRIC_EXPORT_INTERVAL
+          value: "1"

config/operator/e2e-test/otel/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- deployment.yaml

config/operator/kustomization.yaml

@@ -5,6 +5,7 @@ resources:
 - role.yaml
 - role_binding.yaml
 - service_account.yaml
+- metrics.service.yaml
 labels:
 - includeSelectors: true
   includeTemplates: true

config/operator/metrics.service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: operator-metrics
+spec:
+  type: ClusterIP
+  ports:
+  - name: metrics
+    protocol: TCP
+    port: 2223
+    targetPort: metrics

tests/checks/operator_otel_metrics/operator_otel_metrics_test.go

@@ -0,0 +1,241 @@
+//go:build e2e
+// +build e2e
+
+package operator_otel_metrics_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	prommodel "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/client-go/kubernetes"
+
+	. "github.com/kedacore/http-add-on/tests/helper"
+)
+
+const (
+	testName = "operator-otel-metrics-test"
+)
+
+var (
+	testNamespace        = fmt.Sprintf("%s-ns", testName)
+	deploymentName       = fmt.Sprintf("%s-deployment", testName)
+	serviceName          = fmt.Sprintf("%s-service", testName)
+	clientName           = fmt.Sprintf("%s-client", testName)
+	httpScaledObjectName = fmt.Sprintf("%s-http-so", testName)
+	host                 = testName
+	minReplicaCount      = 0
+	maxReplicaCount      = 1
+	otelCollectorPromURL = "http://opentelemetry-collector.open-telemetry-system:8889/metrics"
+)
+
+type templateData struct {
+	TestNamespace        string
+	DeploymentName       string
+	ServiceName          string
+	ClientName           string
+	HTTPScaledObjectName string
+	Host                 string
+	MinReplicas          int
+	MaxReplicas          int
+}
+
+const (
+	serviceTemplate = `
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{.ServiceName}}
+  namespace: {{.TestNamespace}}
+  labels:
+    app: {{.DeploymentName}}
+spec:
+  ports:
+    - port: 8080
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: {{.DeploymentName}}
+`
+
+	deploymentTemplate = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{.DeploymentName}}
+  namespace: {{.TestNamespace}}
+  labels:
+    app: {{.DeploymentName}}
+spec:
+  replicas: 0
+  selector:
+    matchLabels:
+      app: {{.DeploymentName}}
+  template:
+    metadata:
+      labels:
+        app: {{.DeploymentName}}
+    spec:
+      containers:
+        - name: {{.DeploymentName}}
+          image: registry.k8s.io/e2e-test-images/agnhost:2.45
+          args:
+          - netexec
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+`
+
+	loadJobTemplate = `
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: generate-request
+  namespace: {{.TestNamespace}}
+spec:
+  template:
+    spec:
+      containers:
+      - name: curl-client
+        image: curlimages/curl
+        imagePullPolicy: Always
+        command: ["curl", "-H", "Host: {{.Host}}", "keda-add-ons-http-interceptor-proxy.keda:8080"]
+      restartPolicy: Never
+  activeDeadlineSeconds: 600
+  backoffLimit: 5
+`
+
+	httpScaledObjectTemplate = `
+kind: HTTPScaledObject
+apiVersion: http.keda.sh/v1alpha1
+metadata:
+  name: {{.HTTPScaledObjectName}}
+  namespace: {{.TestNamespace}}
+spec:
+  hosts:
+  - {{.Host}}
+  targetPendingRequests: 100
+  scaledownPeriod: 10
+  scaleTargetRef:
+    name: {{.DeploymentName}}
+    service: {{.ServiceName}}
+    port: 8080
+  replicas:
+    min: {{ .MinReplicas }}
+    max: {{ .MaxReplicas }}
+`
+
+	clientTemplate = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{.ClientName}}
+  namespace: {{.TestNamespace}}
+spec:
+  containers:
+  - name: {{.ClientName}}
+    image: curlimages/curl
+    command:
+      - sh
+      - -c
+      - "exec tail -f /dev/null"`
+)
+
+func TestMetricGeneration(t *testing.T) {
+	// setup
+	t.Log("--- setting up ---")
+	// Create kubernetes resources
+	kc := GetKubernetesClient(t)
+	data, templates := getTemplateData()
+	CreateKubernetesResources(t, kc, testNamespace, data, templates)
+
+	assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 6, 10),
+		"replica count should be %d after 1 minutes", minReplicaCount)
+
+	// Send a test request to the interceptor
+	sendLoad(t, kc, data)
+
+	// Fetch metrics and validate them
+	family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL))
+	val, ok := family["operator_http_scaled_object_count_total"]
+	// If the metric is not found first time around then retry with a delay.
+	if !ok {
+		// Add a small sleep to allow metrics to be pushed from the exporter to the collector
+		time.Sleep(10 * time.Second)
+		// Fetch metrics and validate them
+		family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL))
+		val, ok = family["operator_http_scaled_object_count_total"]
+	}
+	assert.True(t, ok, "operator_http_scaled_object_count_total is available")
+
+	requestCount := getMetricsValue(val)
+	assert.GreaterOrEqual(t, requestCount, float64(1))
+
+	// cleanup
+	DeleteKubernetesResources(t, testNamespace, data, templates)
+}
+
+func sendLoad(t *testing.T, kc *kubernetes.Clientset, data templateData) {
+	t.Log("--- sending load ---")
+
+	KubectlApplyWithTemplate(t, data, "loadJobTemplate", loadJobTemplate)
+
+	assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 6, 10),
+		"replica count should be %d after 1 minutes", maxReplicaCount)
+}
+
+func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*prommodel.MetricFamily {
+	out, _, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd)
+	assert.NoErrorf(t, err, "cannot execute command - %s", err)
+
+	parser := expfmt.TextParser{}
+	// Ensure EOL
+	reader := strings.NewReader(strings.ReplaceAll(out, "\r\n", "\n"))
+	families, err := parser.TextToMetricFamilies(reader)
+	assert.NoErrorf(t, err, "cannot parse metrics - %s", err)
+
+	return families
+}
+
+func getMetricsValue(val *prommodel.MetricFamily) float64 {
+	if val.GetName() == "operator_http_scaled_object_count_total" {
+		metrics := val.GetMetric()
+		for _, metric := range metrics {
+			labels := metric.GetLabel()
+			for _, label := range labels {
+				if *label.Name == "namespace" && *label.Value == testNamespace {
+					return metric.GetCounter().GetValue()
+				}
+			}
+		}
+	}
+	return 0
+}
+
+func getTemplateData() (templateData, []Template) {
+	return templateData{
+			TestNamespace:        testNamespace,
+			DeploymentName:       deploymentName,
+			ServiceName:          serviceName,
+			ClientName:           clientName,
+			HTTPScaledObjectName: httpScaledObjectName,
+			Host:                 host,
+			MinReplicas:          minReplicaCount,
+			MaxReplicas:          maxReplicaCount,
+		}, []Template{
+			{Name: "deploymentTemplate", Config: deploymentTemplate},
+			{Name: "serviceNameTemplate", Config: serviceTemplate},
+			{Name: "clientTemplate", Config: clientTemplate},
+			{Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate},
+		}
+}

tests/checks/operator_prometheus_metrics/operator_prometheus_metrics_test.go

@@ -0,0 +1,236 @@
+//go:build e2e
+// +build e2e
+
+package operator_prometheus_metrics_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	prommodel "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/client-go/kubernetes"
+
+	. "github.com/kedacore/http-add-on/tests/helper"
+)
+
+const (
+	testName = "operator-prom-metrics-test"
+)
+
+var (
+	testNamespace             = fmt.Sprintf("%s-ns", testName)
+	deploymentName            = fmt.Sprintf("%s-deployment", testName)
+	serviceName               = fmt.Sprintf("%s-service", testName)
+	clientName                = fmt.Sprintf("%s-client", testName)
+	httpScaledObjectName      = fmt.Sprintf("%s-http-so", testName)
+	host                      = testName
+	minReplicaCount           = 0
+	maxReplicaCount           = 1
+	kedaOperatorPrometheusURL = "http://keda-add-ons-http-operator-metrics.keda:2223/metrics"
+)
+
+type templateData struct {
+	TestNamespace        string
+	DeploymentName       string
+	ServiceName          string
+	ClientName           string
+	HTTPScaledObjectName string
+	Host                 string
+	MinReplicas          int
+	MaxReplicas          int
+}
+
+const (
+	serviceTemplate = `
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{.ServiceName}}
+  namespace: {{.TestNamespace}}
+  labels:
+    app: {{.DeploymentName}}
+spec:
+  ports:
+    - port: 8080
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: {{.DeploymentName}}
+`
+
+	deploymentTemplate = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{.DeploymentName}}
+  namespace: {{.TestNamespace}}
+  labels:
+    app: {{.DeploymentName}}
+spec:
+  replicas: 0
+  selector:
+    matchLabels:
+      app: {{.DeploymentName}}
+  template:
+    metadata:
+      labels:
+        app: {{.DeploymentName}}
+    spec:
+      containers:
+        - name: {{.DeploymentName}}
+          image: registry.k8s.io/e2e-test-images/agnhost:2.45
+          args:
+          - netexec
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+`
+
+	loadJobTemplate = `
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: generate-request
+  namespace: {{.TestNamespace}}
+spec:
+  template:
+    spec:
+      containers:
+      - name: curl-client
+        image: curlimages/curl
+        imagePullPolicy: Always
+        command: ["curl", "-H", "Host: {{.Host}}", "keda-add-ons-http-interceptor-proxy.keda:8080"]
+      restartPolicy: Never
+  activeDeadlineSeconds: 600
+  backoffLimit: 5
+`
+
+	httpScaledObjectTemplate = `
+kind: HTTPScaledObject
+apiVersion: http.keda.sh/v1alpha1
+metadata:
+  name: {{.HTTPScaledObjectName}}
+  namespace: {{.TestNamespace}}
+spec:
+  hosts:
+  - {{.Host}}
+  targetPendingRequests: 100
+  scaledownPeriod: 10
+  scaleTargetRef:
+    name: {{.DeploymentName}}
+    service: {{.ServiceName}}
+    port: 8080
+  replicas:
+    min: {{ .MinReplicas }}
+    max: {{ .MaxReplicas }}
+`
+
+	clientTemplate = `
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{.ClientName}}
+  namespace: {{.TestNamespace}}
+spec:
+  containers:
+  - name: {{.ClientName}}
+    image: curlimages/curl
+    command:
+      - sh
+      - -c
+      - "exec tail -f /dev/null"`
+)
+
+func TestMetricGeneration(t *testing.T) {
+	// setup
+	t.Log("--- setting up ---")
+	// Create kubernetes resources
+	kc := GetKubernetesClient(t)
+	data, templates := getTemplateData()
+	CreateKubernetesResources(t, kc, testNamespace, data, templates)
+
+	assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 6, 10),
+		"replica count should be %d after 1 minutes", minReplicaCount)
+	t.Log("--- ASSERT replicas ---")
+
+	// Send a test request to the interceptor
+	sendLoad(t, kc, data)
+
+	// Fetch metrics and validate them
+	family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
+	val, ok := family["operator_http_scaled_object_count_total"]
+	assert.True(t, ok, "operator_http_scaled_object_count_total is available")
+	t.Log("--- ASSERT operator_http_scaled_object_count_total is available ---")
+
+	requestCount := getMetricsValue(val)
+	assert.GreaterOrEqual(t, requestCount, float64(1))
+	t.Log("--- ASSERT metrics greater than1 ---")
+
+	// cleanup
+	DeleteKubernetesResources(t, testNamespace, data, templates)
+}
+
+func sendLoad(t *testing.T, kc *kubernetes.Clientset, data templateData) {
+	t.Log("--- sending load ---")
+
+	KubectlApplyWithTemplate(t, data, "loadJobTemplate", loadJobTemplate)
+
+	assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 6, 10),
+		"replica count should be %d after 1 minutes", maxReplicaCount)
+}
+
+func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*prommodel.MetricFamily {
+	out, _, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd)
+	assert.NoErrorf(t, err, "cannot execute command - %s", err)
+	t.Logf("OUTPUT - %s", out)
+
+	parser := expfmt.TextParser{}
+	// Ensure EOL
+	reader := strings.NewReader(strings.ReplaceAll(out, "\r\n", "\n"))
+	families, err := parser.TextToMetricFamilies(reader)
+	assert.NoErrorf(t, err, "cannot parse metrics - %s", err)
+
+	return families
+}
+
+func getMetricsValue(val *prommodel.MetricFamily) float64 {
+	if val.GetName() == "operator_http_scaled_object_count_total" {
+		metrics := val.GetMetric()
+		for _, metric := range metrics {
+			labels := metric.GetLabel()
+			for _, label := range labels {
+				if *label.Name == "namespace" && *label.Value == testNamespace {
+					return metric.GetCounter().GetValue()
+				}
+			}
+		}
+	}
+	return 0
+}
+
+func getTemplateData() (templateData, []Template) {
+	return templateData{
+			TestNamespace:        testNamespace,
+			DeploymentName:       deploymentName,
+			ServiceName:          serviceName,
+			ClientName:           clientName,
+			HTTPScaledObjectName: httpScaledObjectName,
+			Host:                 host,
+			MinReplicas:          minReplicaCount,
+			MaxReplicas:          maxReplicaCount,
+		}, []Template{
+			{Name: "deploymentTemplate", Config: deploymentTemplate},
+			{Name: "serviceNameTemplate", Config: serviceTemplate},
+			{Name: "clientTemplate", Config: clientTemplate},
+			{Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate},
+		}
+}