add e2e tests and update config/operator to suport e2e
Signed-off-by: Elieser Pereira <elieser.pereiraa@gmail.com>
This commit is contained in:
Elieser Pereira
parent
c7c771703e
commit
ff77d802ee
3
Makefile
3
Makefile
|
|
@ -221,6 +221,9 @@ deploy: manifests kustomize ## Deploy to the K8s cluster specified in ~/.kube/co
|
|||
cd config/operator && \
|
||||
$(KUSTOMIZE) edit set image ghcr.io/kedacore/http-add-on-operator=${IMAGE_OPERATOR_VERSIONED_TAG}
|
||||
|
||||
cd config/operator && \
|
||||
$(KUSTOMIZE) edit add patch --path e2e-test/otel/deployment.yaml --group apps --kind Deployment --name operator --version v1
|
||||
|
||||
$(KUSTOMIZE) build config/default | kubectl apply -f -
|
||||
|
||||
undeploy:
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ spec:
|
|||
value: ""
|
||||
ports:
|
||||
- name: metrics
|
||||
containerPort: 8080
|
||||
containerPort: 2223
|
||||
- name: probes
|
||||
containerPort: 8081
|
||||
livenessProbe:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: operator
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: operator
|
||||
env:
|
||||
- name: OTEL_PROM_EXPORTER_ENABLED
|
||||
value: "true"
|
||||
- name: OTEL_PROM_EXPORTER_PORT
|
||||
value: "2223"
|
||||
- name: OTEL_EXPORTER_OTLP_METRICS_ENABLED
|
||||
value: "true"
|
||||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||||
value: "http://opentelemetry-collector.open-telemetry-system:4318"
|
||||
- name: OTEL_METRIC_EXPORT_INTERVAL
|
||||
value: "1"
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- deployment.yaml
|
||||
|
|
@ -5,6 +5,7 @@ resources:
|
|||
- role.yaml
|
||||
- role_binding.yaml
|
||||
- service_account.yaml
|
||||
- metrics.service.yaml
|
||||
labels:
|
||||
- includeSelectors: true
|
||||
includeTemplates: true
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: operator-metrics
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: metrics
|
||||
protocol: TCP
|
||||
port: 2223
|
||||
targetPort: metrics
|
||||
|
|
@ -0,0 +1,241 @@
|
|||
//go:build e2e
|
||||
// +build e2e
|
||||
|
||||
package operator_otel_metrics_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
prommodel "github.com/prometheus/client_model/go"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
. "github.com/kedacore/http-add-on/tests/helper"
|
||||
)
|
||||
|
||||
const (
|
||||
testName = "operator-otel-metrics-test"
|
||||
)
|
||||
|
||||
var (
|
||||
testNamespace = fmt.Sprintf("%s-ns", testName)
|
||||
deploymentName = fmt.Sprintf("%s-deployment", testName)
|
||||
serviceName = fmt.Sprintf("%s-service", testName)
|
||||
clientName = fmt.Sprintf("%s-client", testName)
|
||||
httpScaledObjectName = fmt.Sprintf("%s-http-so", testName)
|
||||
host = testName
|
||||
minReplicaCount = 0
|
||||
maxReplicaCount = 1
|
||||
otelCollectorPromURL = "http://opentelemetry-collector.open-telemetry-system:8889/metrics"
|
||||
)
|
||||
|
||||
type templateData struct {
|
||||
TestNamespace string
|
||||
DeploymentName string
|
||||
ServiceName string
|
||||
ClientName string
|
||||
HTTPScaledObjectName string
|
||||
Host string
|
||||
MinReplicas int
|
||||
MaxReplicas int
|
||||
}
|
||||
|
||||
const (
|
||||
serviceTemplate = `
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{.ServiceName}}
|
||||
namespace: {{.TestNamespace}}
|
||||
labels:
|
||||
app: {{.DeploymentName}}
|
||||
spec:
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
app: {{.DeploymentName}}
|
||||
`
|
||||
|
||||
deploymentTemplate = `
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{.DeploymentName}}
|
||||
namespace: {{.TestNamespace}}
|
||||
labels:
|
||||
app: {{.DeploymentName}}
|
||||
spec:
|
||||
replicas: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{.DeploymentName}}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{.DeploymentName}}
|
||||
spec:
|
||||
containers:
|
||||
- name: {{.DeploymentName}}
|
||||
image: registry.k8s.io/e2e-test-images/agnhost:2.45
|
||||
args:
|
||||
- netexec
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
`
|
||||
|
||||
loadJobTemplate = `
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: generate-request
|
||||
namespace: {{.TestNamespace}}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: curl-client
|
||||
image: curlimages/curl
|
||||
imagePullPolicy: Always
|
||||
command: ["curl", "-H", "Host: {{.Host}}", "keda-add-ons-http-interceptor-proxy.keda:8080"]
|
||||
restartPolicy: Never
|
||||
activeDeadlineSeconds: 600
|
||||
backoffLimit: 5
|
||||
`
|
||||
|
||||
httpScaledObjectTemplate = `
|
||||
kind: HTTPScaledObject
|
||||
apiVersion: http.keda.sh/v1alpha1
|
||||
metadata:
|
||||
name: {{.HTTPScaledObjectName}}
|
||||
namespace: {{.TestNamespace}}
|
||||
spec:
|
||||
hosts:
|
||||
- {{.Host}}
|
||||
targetPendingRequests: 100
|
||||
scaledownPeriod: 10
|
||||
scaleTargetRef:
|
||||
name: {{.DeploymentName}}
|
||||
service: {{.ServiceName}}
|
||||
port: 8080
|
||||
replicas:
|
||||
min: {{ .MinReplicas }}
|
||||
max: {{ .MaxReplicas }}
|
||||
`
|
||||
|
||||
clientTemplate = `
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: {{.ClientName}}
|
||||
namespace: {{.TestNamespace}}
|
||||
spec:
|
||||
containers:
|
||||
- name: {{.ClientName}}
|
||||
image: curlimages/curl
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- "exec tail -f /dev/null"`
|
||||
)
|
||||
|
||||
func TestMetricGeneration(t *testing.T) {
|
||||
// setup
|
||||
t.Log("--- setting up ---")
|
||||
// Create kubernetes resources
|
||||
kc := GetKubernetesClient(t)
|
||||
data, templates := getTemplateData()
|
||||
CreateKubernetesResources(t, kc, testNamespace, data, templates)
|
||||
|
||||
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 6, 10),
|
||||
"replica count should be %d after 1 minutes", minReplicaCount)
|
||||
|
||||
// Send a test request to the interceptor
|
||||
sendLoad(t, kc, data)
|
||||
|
||||
// Fetch metrics and validate them
|
||||
family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL))
|
||||
val, ok := family["operator_http_scaled_object_count_total"]
|
||||
// If the metric is not found first time around then retry with a delay.
|
||||
if !ok {
|
||||
// Add a small sleep to allow metrics to be pushed from the exporter to the collector
|
||||
time.Sleep(10 * time.Second)
|
||||
// Fetch metrics and validate them
|
||||
family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", otelCollectorPromURL))
|
||||
val, ok = family["operator_http_scaled_object_count_total"]
|
||||
}
|
||||
assert.True(t, ok, "operator_http_scaled_object_count_total is available")
|
||||
|
||||
requestCount := getMetricsValue(val)
|
||||
assert.GreaterOrEqual(t, requestCount, float64(1))
|
||||
|
||||
// cleanup
|
||||
DeleteKubernetesResources(t, testNamespace, data, templates)
|
||||
}
|
||||
|
||||
func sendLoad(t *testing.T, kc *kubernetes.Clientset, data templateData) {
|
||||
t.Log("--- sending load ---")
|
||||
|
||||
KubectlApplyWithTemplate(t, data, "loadJobTemplate", loadJobTemplate)
|
||||
|
||||
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 6, 10),
|
||||
"replica count should be %d after 1 minutes", maxReplicaCount)
|
||||
}
|
||||
|
||||
func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*prommodel.MetricFamily {
|
||||
out, _, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd)
|
||||
assert.NoErrorf(t, err, "cannot execute command - %s", err)
|
||||
|
||||
parser := expfmt.TextParser{}
|
||||
// Ensure EOL
|
||||
reader := strings.NewReader(strings.ReplaceAll(out, "\r\n", "\n"))
|
||||
families, err := parser.TextToMetricFamilies(reader)
|
||||
assert.NoErrorf(t, err, "cannot parse metrics - %s", err)
|
||||
|
||||
return families
|
||||
}
|
||||
|
||||
func getMetricsValue(val *prommodel.MetricFamily) float64 {
|
||||
if val.GetName() == "operator_http_scaled_object_count_total" {
|
||||
metrics := val.GetMetric()
|
||||
for _, metric := range metrics {
|
||||
labels := metric.GetLabel()
|
||||
for _, label := range labels {
|
||||
if *label.Name == "namespace" && *label.Value == testNamespace {
|
||||
return metric.GetCounter().GetValue()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func getTemplateData() (templateData, []Template) {
|
||||
return templateData{
|
||||
TestNamespace: testNamespace,
|
||||
DeploymentName: deploymentName,
|
||||
ServiceName: serviceName,
|
||||
ClientName: clientName,
|
||||
HTTPScaledObjectName: httpScaledObjectName,
|
||||
Host: host,
|
||||
MinReplicas: minReplicaCount,
|
||||
MaxReplicas: maxReplicaCount,
|
||||
}, []Template{
|
||||
{Name: "deploymentTemplate", Config: deploymentTemplate},
|
||||
{Name: "serviceNameTemplate", Config: serviceTemplate},
|
||||
{Name: "clientTemplate", Config: clientTemplate},
|
||||
{Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate},
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,236 @@
|
|||
//go:build e2e
|
||||
// +build e2e
|
||||
|
||||
package operator_prometheus_metrics_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
prommodel "github.com/prometheus/client_model/go"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
. "github.com/kedacore/http-add-on/tests/helper"
|
||||
)
|
||||
|
||||
const (
|
||||
testName = "operator-prom-metrics-test"
|
||||
)
|
||||
|
||||
var (
|
||||
testNamespace = fmt.Sprintf("%s-ns", testName)
|
||||
deploymentName = fmt.Sprintf("%s-deployment", testName)
|
||||
serviceName = fmt.Sprintf("%s-service", testName)
|
||||
clientName = fmt.Sprintf("%s-client", testName)
|
||||
httpScaledObjectName = fmt.Sprintf("%s-http-so", testName)
|
||||
host = testName
|
||||
minReplicaCount = 0
|
||||
maxReplicaCount = 1
|
||||
kedaOperatorPrometheusURL = "http://keda-add-ons-http-operator-metrics.keda:2223/metrics"
|
||||
)
|
||||
|
||||
type templateData struct {
|
||||
TestNamespace string
|
||||
DeploymentName string
|
||||
ServiceName string
|
||||
ClientName string
|
||||
HTTPScaledObjectName string
|
||||
Host string
|
||||
MinReplicas int
|
||||
MaxReplicas int
|
||||
}
|
||||
|
||||
const (
|
||||
serviceTemplate = `
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{.ServiceName}}
|
||||
namespace: {{.TestNamespace}}
|
||||
labels:
|
||||
app: {{.DeploymentName}}
|
||||
spec:
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: http
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
app: {{.DeploymentName}}
|
||||
`
|
||||
|
||||
deploymentTemplate = `
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{.DeploymentName}}
|
||||
namespace: {{.TestNamespace}}
|
||||
labels:
|
||||
app: {{.DeploymentName}}
|
||||
spec:
|
||||
replicas: 0
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{.DeploymentName}}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{.DeploymentName}}
|
||||
spec:
|
||||
containers:
|
||||
- name: {{.DeploymentName}}
|
||||
image: registry.k8s.io/e2e-test-images/agnhost:2.45
|
||||
args:
|
||||
- netexec
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: http
|
||||
`
|
||||
|
||||
loadJobTemplate = `
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: generate-request
|
||||
namespace: {{.TestNamespace}}
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: curl-client
|
||||
image: curlimages/curl
|
||||
imagePullPolicy: Always
|
||||
command: ["curl", "-H", "Host: {{.Host}}", "keda-add-ons-http-interceptor-proxy.keda:8080"]
|
||||
restartPolicy: Never
|
||||
activeDeadlineSeconds: 600
|
||||
backoffLimit: 5
|
||||
`
|
||||
|
||||
httpScaledObjectTemplate = `
|
||||
kind: HTTPScaledObject
|
||||
apiVersion: http.keda.sh/v1alpha1
|
||||
metadata:
|
||||
name: {{.HTTPScaledObjectName}}
|
||||
namespace: {{.TestNamespace}}
|
||||
spec:
|
||||
hosts:
|
||||
- {{.Host}}
|
||||
targetPendingRequests: 100
|
||||
scaledownPeriod: 10
|
||||
scaleTargetRef:
|
||||
name: {{.DeploymentName}}
|
||||
service: {{.ServiceName}}
|
||||
port: 8080
|
||||
replicas:
|
||||
min: {{ .MinReplicas }}
|
||||
max: {{ .MaxReplicas }}
|
||||
`
|
||||
|
||||
clientTemplate = `
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: {{.ClientName}}
|
||||
namespace: {{.TestNamespace}}
|
||||
spec:
|
||||
containers:
|
||||
- name: {{.ClientName}}
|
||||
image: curlimages/curl
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- "exec tail -f /dev/null"`
|
||||
)
|
||||
|
||||
func TestMetricGeneration(t *testing.T) {
|
||||
// setup
|
||||
t.Log("--- setting up ---")
|
||||
// Create kubernetes resources
|
||||
kc := GetKubernetesClient(t)
|
||||
data, templates := getTemplateData()
|
||||
CreateKubernetesResources(t, kc, testNamespace, data, templates)
|
||||
|
||||
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, minReplicaCount, 6, 10),
|
||||
"replica count should be %d after 1 minutes", minReplicaCount)
|
||||
t.Log("--- ASSERT replicas ---")
|
||||
|
||||
// Send a test request to the interceptor
|
||||
sendLoad(t, kc, data)
|
||||
|
||||
// Fetch metrics and validate them
|
||||
family := fetchAndParsePrometheusMetrics(t, fmt.Sprintf("curl --insecure %s", kedaOperatorPrometheusURL))
|
||||
val, ok := family["operator_http_scaled_object_count_total"]
|
||||
assert.True(t, ok, "operator_http_scaled_object_count_total is available")
|
||||
t.Log("--- ASSERT operator_http_scaled_object_count_total is available ---")
|
||||
|
||||
requestCount := getMetricsValue(val)
|
||||
assert.GreaterOrEqual(t, requestCount, float64(1))
|
||||
t.Log("--- ASSERT metrics greater than1 ---")
|
||||
|
||||
// cleanup
|
||||
DeleteKubernetesResources(t, testNamespace, data, templates)
|
||||
}
|
||||
|
||||
func sendLoad(t *testing.T, kc *kubernetes.Clientset, data templateData) {
|
||||
t.Log("--- sending load ---")
|
||||
|
||||
KubectlApplyWithTemplate(t, data, "loadJobTemplate", loadJobTemplate)
|
||||
|
||||
assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 6, 10),
|
||||
"replica count should be %d after 1 minutes", maxReplicaCount)
|
||||
}
|
||||
|
||||
func fetchAndParsePrometheusMetrics(t *testing.T, cmd string) map[string]*prommodel.MetricFamily {
|
||||
out, _, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, cmd)
|
||||
assert.NoErrorf(t, err, "cannot execute command - %s", err)
|
||||
t.Logf("OUTPUT - %s", out)
|
||||
|
||||
parser := expfmt.TextParser{}
|
||||
// Ensure EOL
|
||||
reader := strings.NewReader(strings.ReplaceAll(out, "\r\n", "\n"))
|
||||
families, err := parser.TextToMetricFamilies(reader)
|
||||
assert.NoErrorf(t, err, "cannot parse metrics - %s", err)
|
||||
|
||||
return families
|
||||
}
|
||||
|
||||
func getMetricsValue(val *prommodel.MetricFamily) float64 {
|
||||
if val.GetName() == "operator_http_scaled_object_count_total" {
|
||||
metrics := val.GetMetric()
|
||||
for _, metric := range metrics {
|
||||
labels := metric.GetLabel()
|
||||
for _, label := range labels {
|
||||
if *label.Name == "namespace" && *label.Value == testNamespace {
|
||||
return metric.GetCounter().GetValue()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func getTemplateData() (templateData, []Template) {
|
||||
return templateData{
|
||||
TestNamespace: testNamespace,
|
||||
DeploymentName: deploymentName,
|
||||
ServiceName: serviceName,
|
||||
ClientName: clientName,
|
||||
HTTPScaledObjectName: httpScaledObjectName,
|
||||
Host: host,
|
||||
MinReplicas: minReplicaCount,
|
||||
MaxReplicas: maxReplicaCount,
|
||||
}, []Template{
|
||||
{Name: "deploymentTemplate", Config: deploymentTemplate},
|
||||
{Name: "serviceNameTemplate", Config: serviceTemplate},
|
||||
{Name: "clientTemplate", Config: clientTemplate},
|
||||
{Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate},
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue