feat: Adding support to the interceptor proxy for TLS on the wire (#928)

* feat: Adding support to the interceptor proxy for TLS on the wire

Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com>

* chore: adding e2e tests and tidying up changes

Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com>

* chore: changing interceptor tls port for e2e tests

Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com>

---------

Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com>
This commit is contained in:
zorocloud 2024-05-01 18:47:18 +01:00 committed by GitHub
parent e3d2e81220
commit 38f50bfb0a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 906 additions and 16 deletions

7
.gitignore vendored
View File

@ -355,3 +355,10 @@ admin/Cargo.lock
/target
.envrc
# locally generated certs for testing TLS
*.crt
*.pem
*.csr
*.srl
*.ext

View File

@ -29,7 +29,7 @@ This changelog keeps track of work items that have been completed and are ready
### Improvements
- **General**: TODO ([#TODO](https://github.com/kedacore/http-add-on/issues/TODO))
- **General**: Add configurable TLS on the wire support to the interceptor proxy ([#907](https://github.com/kedacore/http-add-on/issues/907))
### Fixes

View File

@ -32,6 +32,17 @@ GO_LDFLAGS="-X github.com/kedacore/http-add-on/pkg/build.version=${VERSION} -X g
GIT_COMMIT ?= $(shell git rev-list -1 HEAD)
GIT_COMMIT_SHORT ?= $(shell git rev-parse --short HEAD)
define DOMAINS
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
subjectAltName = @alt_names
[alt_names]
DNS.1 = localhost
DNS.2 = *.keda
DNS.3 = *.interceptor-tls-test-ns
endef
export DOMAINS
# Build targets
build-operator:
@ -45,8 +56,22 @@ build-scaler:
build: build-operator build-interceptor build-scaler
# generate certs for local unit and e2e tests
rootca-test-certs:
mkdir -p certs
openssl req -x509 -nodes -new -sha256 -days 1024 -newkey rsa:2048 -keyout certs/RootCA.key -out certs/RootCA.pem -subj "/C=US/CN=Keda-Root-CA"
openssl x509 -outform pem -in certs/RootCA.pem -out certs/RootCA.crt
test-certs: rootca-test-certs
echo "$$DOMAINS" > certs/domains.ext
openssl req -new -nodes -newkey rsa:2048 -keyout certs/tls.key -out certs/tls.csr -subj "/C=US/ST=KedaState/L=KedaCity/O=Keda-Certificates/CN=keda.local"
openssl x509 -req -sha256 -days 1024 -in certs/tls.csr -CA certs/RootCA.pem -CAkey certs/RootCA.key -CAcreateserial -extfile certs/domains.ext -out certs/tls.crt
clean-test-certs:
rm -r certs || true
# Test targets
test: fmt vet
test: fmt vet test-certs
go test ./...
e2e-test:
@ -156,6 +181,12 @@ deploy: manifests kustomize ## Deploy to the K8s cluster specified in ~/.kube/co
cd config/interceptor && \
$(KUSTOMIZE) edit add patch --path e2e-test/scaledobject.yaml --group keda.sh --kind ScaledObject --name interceptor --version v1alpha1
cd config/interceptor && \
$(KUSTOMIZE) edit add patch --path tls/deployment.yaml --group apps --kind Deployment --name interceptor --version v1
cd config/interceptor && \
$(KUSTOMIZE) edit add patch --path tls/proxy.service.yaml --kind Service --name interceptor-proxy --version v1
cd config/scaler && \
$(KUSTOMIZE) edit set image ghcr.io/kedacore/http-add-on-scaler=${IMAGE_SCALER_VERSIONED_TAG}
@ -174,3 +205,8 @@ kind-load:
kind load docker-image ghcr.io/kedacore/http-add-on-operator:${VERSION}
kind load docker-image ghcr.io/kedacore/http-add-on-interceptor:${VERSION}
kind load docker-image ghcr.io/kedacore/http-add-on-scaler:${VERSION}
k3d-import:
k3d image import ghcr.io/kedacore/http-add-on-operator:main
k3d image import ghcr.io/kedacore/http-add-on-interceptor:main
k3d image import ghcr.io/kedacore/http-add-on-scaler:main

View File

@ -0,0 +1,30 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: interceptor
spec:
replicas: 1
template:
spec:
containers:
- name: interceptor
ports:
- name: proxy-tls
containerPort: 8443
env:
- name: KEDA_HTTP_PROXY_TLS_ENABLED
value: "true"
- name: KEDA_HTTP_PROXY_TLS_CERT_PATH
value: "/certs/tls.crt"
- name: KEDA_HTTP_PROXY_TLS_KEY_PATH
value: "/certs/tls.key"
- name: KEDA_HTTP_PROXY_TLS_PORT
value: "8443"
volumeMounts:
- readOnly: true
mountPath: "/certs"
name: certs
volumes:
- name: certs
secret:
secretName: keda-tls

View File

@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- deployment.yaml
- proxy.service.yaml

View File

@ -0,0 +1,11 @@
apiVersion: v1
kind: Service
metadata:
name: interceptor-proxy
spec:
type: ClusterIP
ports:
- name: proxy-tls
protocol: TCP
port: 8443
targetPort: proxy-tls

View File

@ -19,3 +19,9 @@ The OTEL exporter can be enabled by setting the `KEDA_HTTP_OTEL_HTTP_EXPORTER_EN
If the collector is exposed on a unsecured endpoint then you can set the `KEDA_HTTP_OTEL_HTTP_COLLECTOR_INSECURE` environment variable to `true` (`false` by default) which will disable client security on the exporter.
If you need to provide any headers such as authentication details in order to utilise your OTEL collector you can add them into the `KEDA_HTTP_OTEL_HTTP_HEADERS` environment variable. The frequency at which the metrics are exported can be configured by setting `KEDA_HTTP_OTEL_METRIC_EXPORT_INTERVAL` to the number of seconds you require between each export interval (`30` by default).
# Configuring TLS for the KEDA HTTP Add-on interceptor proxy
The interceptor proxy has the ability to run both a HTTP and HTTPS server simultaneously to allow you to scale workloads that use either protocol. By default, the interceptor proxy will only serve over HTTP, but this behavior can be changed by configuring the appropriate environment variables on the deployment.
The TLS server can be enabled by setting the environment variable `KEDA_HTTP_PROXY_TLS_ENABLED` to `true` on the interceptor deployment (`false` by default). The TLS server will start on port `8443` by default, but this can be configured by setting `KEDA_HTTP_PROXY_TLS_PORT` to your desired port number. The TLS server will require valid TLS certificates to start, the path to the certificates can be configured via the `KEDA_HTTP_PROXY_TLS_CERT_PATH` and `KEDA_HTTP_PROXY_TLS_KEY_PATH` environment variables (`/certs/tls.crt` and `/certs/tls.key` by default).

View File

@ -34,6 +34,15 @@ type Serving struct {
//
// This is the interval (in milliseconds) representing how often to do a fetch
EndpointsCachePollIntervalMS int `envconfig:"KEDA_HTTP_ENDPOINTS_CACHE_POLLING_INTERVAL_MS" default:"250"`
// ProxyTLSEnabled is a flag to specify whether the interceptor proxy should
// be running using a TLS enabled server
ProxyTLSEnabled bool `envconfig:"KEDA_HTTP_PROXY_TLS_ENABLED" default:"false"`
// TLSCertPath is the path to read the certificate file from for the TLS server
TLSCertPath string `envconfig:"KEDA_HTTP_PROXY_TLS_CERT_PATH" default:"/certs/tls.crt"`
// TLSKeyPath is the path to read the private key file from for the TLS server
TLSKeyPath string `envconfig:"KEDA_HTTP_PROXY_TLS_KEY_PATH" default:"/certs/tls.key"`
// TLSPort is the port that the server should serve on if TLS is enabled
TLSPort int `envconfig:"KEDA_HTTP_PROXY_TLS_PORT" default:"8443"`
}
// Parse parses standard configs using envconfig and returns a pointer to the

View File

@ -2,6 +2,8 @@ package main
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"flag"
"fmt"
@ -68,6 +70,7 @@ func main() {
proxyPort := servingCfg.ProxyPort
adminPort := servingCfg.AdminPort
proxyTLSEnabled := servingCfg.ProxyTLSEnabled
// setup the configured metrics collectors
metrics.NewMetricsCollectors(metricsCfg)
@ -160,12 +163,29 @@ func main() {
})
}
// start the proxy server. this is the server that
// start the proxy servers. This is the server that
// accepts, holds and forwards user requests
// start a proxy server with TLS
if proxyTLSEnabled {
eg.Go(func() error {
setupLog.Info("starting the proxy server", "port", proxyPort)
proxyTLSConfig := map[string]string{"certificatePath": servingCfg.TLSCertPath, "keyPath": servingCfg.TLSKeyPath}
proxyTLSPort := servingCfg.TLSPort
if err := runProxyServer(ctx, ctrl.Log, queues, waitFunc, routingTable, timeoutCfg, proxyPort); !util.IsIgnoredErr(err) {
setupLog.Info("starting the proxy server with TLS enabled", "port", proxyTLSPort)
if err := runProxyServer(ctx, ctrl.Log, queues, waitFunc, routingTable, timeoutCfg, proxyTLSPort, proxyTLSEnabled, proxyTLSConfig); !util.IsIgnoredErr(err) {
setupLog.Error(err, "tls proxy server failed")
return err
}
return nil
})
}
// start a proxy server without TLS.
eg.Go(func() error {
setupLog.Info("starting the proxy server with TLS disabled", "port", proxyPort)
if err := runProxyServer(ctx, ctrl.Log, queues, waitFunc, routingTable, timeoutCfg, proxyPort, false, nil); !util.IsIgnoredErr(err) {
setupLog.Error(err, "proxy server failed")
return err
}
@ -199,7 +219,7 @@ func runAdminServer(
addr := fmt.Sprintf("0.0.0.0:%d", port)
lggr.Info("admin server starting", "address", addr)
return kedahttp.ServeContext(ctx, addr, adminServer)
return kedahttp.ServeContext(ctx, addr, adminServer, false, nil)
}
func runMetricsServer(
@ -209,7 +229,7 @@ func runMetricsServer(
) error {
lggr.Info("starting the prometheus metrics server", "port", metricsCfg.OtelPrometheusExporterPort, "path", "/metrics")
addr := fmt.Sprintf("0.0.0.0:%d", metricsCfg.OtelPrometheusExporterPort)
return kedahttp.ServeContext(ctx, addr, promhttp.Handler())
return kedahttp.ServeContext(ctx, addr, promhttp.Handler(), false, nil)
}
func runProxyServer(
@ -220,6 +240,8 @@ func runProxyServer(
routingTable routing.Table,
timeouts *config.Timeouts,
port int,
tlsEnabled bool,
tlsConfig map[string]string,
) error {
dialer := kedanet.NewNetDialer(timeouts.Connect, timeouts.KeepAlive)
dialContextFunc := kedanet.DialContextWithRetry(dialer, timeouts.DefaultBackoff())
@ -229,12 +251,33 @@ func runProxyServer(
})
go probeHandler.Start(ctx)
tlsCfg := tls.Config{}
if tlsEnabled {
caCert, err := os.ReadFile(tlsConfig["certificatePath"])
if err != nil {
logger.Error(fmt.Errorf("error reading file from TLSCertPath"), "error", err)
os.Exit(1)
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
cert, err := tls.LoadX509KeyPair(tlsConfig["certificatePath"], tlsConfig["keyPath"])
if err != nil {
logger.Error(fmt.Errorf("error creating TLS configuration for proxy server"), "error", err)
os.Exit(1)
}
tlsCfg.RootCAs = caCertPool
tlsCfg.Certificates = []tls.Certificate{cert}
}
var upstreamHandler http.Handler
upstreamHandler = newForwardingHandler(
logger,
dialContextFunc,
waitFunc,
newForwardingConfigFromTimeouts(timeouts),
&tlsCfg,
)
upstreamHandler = middleware.NewCountingMiddleware(
q,
@ -246,6 +289,7 @@ func runProxyServer(
routingTable,
probeHandler,
upstreamHandler,
tlsEnabled,
)
rootHandler = middleware.NewLogging(
logger,
@ -258,5 +302,5 @@ func runProxyServer(
addr := fmt.Sprintf("0.0.0.0:%d", port)
logger.Info("proxy server starting", "address", addr)
return kedahttp.ServeContext(ctx, addr, rootHandler)
return kedahttp.ServeContext(ctx, addr, rootHandler, tlsEnabled, tlsConfig)
}

View File

@ -2,8 +2,11 @@ package main
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"net/http"
"os"
"strconv"
"testing"
"time"
@ -74,6 +77,8 @@ func TestRunProxyServerCountMiddleware(t *testing.T) {
routingTable,
timeouts,
port,
false,
map[string]string{},
)
})
// wait for server to start
@ -147,3 +152,148 @@ func TestRunProxyServerCountMiddleware(t *testing.T) {
done()
r.Error(g.Wait())
}
func TestRunProxyServerWithTLSCountMiddleware(t *testing.T) {
const (
port = 8443
host = "samplehost"
)
r := require.New(t)
ctx, done := context.WithCancel(
context.Background(),
)
defer done()
originHdl := kedanet.NewTestHTTPHandlerWrapper(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}),
)
originSrv, originURL, err := kedanet.StartTestServer(originHdl)
r.NoError(err)
defer originSrv.Close()
originPort, err := strconv.Atoi(originURL.Port())
r.NoError(err)
g, ctx := errgroup.WithContext(ctx)
q := queue.NewFakeCounter()
httpso := targetFromURL(
originURL,
originPort,
"testdepl",
"testsvc",
)
namespacedName := k8s.NamespacedNameFromObject(httpso).String()
// set up a fake host that we can spoof
// when we later send request to the proxy,
// so that the proxy calculates a URL for that
// host that points to the (above) fake origin
// server
routingTable := routingtest.NewTable()
routingTable.Memory[host] = httpso
timeouts := &config.Timeouts{}
waiterCh := make(chan struct{})
waitFunc := func(_ context.Context, _, _ string) (bool, error) {
<-waiterCh
return false, nil
}
g.Go(func() error {
return runProxyServer(
ctx,
logr.Discard(),
q,
waitFunc,
routingTable,
timeouts,
port,
true,
map[string]string{"certificatePath": "../certs/tls.crt", "keyPath": "../certs/tls.key"},
)
})
// wait for server to start
time.Sleep(500 * time.Millisecond)
// make an HTTPs request in the background
g.Go(func() error {
f, err := os.ReadFile("../certs/RootCA.pem")
if err != nil {
t.Errorf("Unable to find RootCA for test, please run tests via `make test`")
}
rootCAs, _ := x509.SystemCertPool()
rootCAs.AppendCertsFromPEM(f)
http.DefaultClient.Transport = &http.Transport{
TLSClientConfig: &tls.Config{RootCAs: rootCAs},
}
req, err := http.NewRequest(
"GET",
fmt.Sprintf(
"https://localhost:%d", port,
), nil,
)
if err != nil {
return err
}
req.Host = host
// Allow us to use our self made certs
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf(
"unexpected status code: %d",
resp.StatusCode,
)
}
if resp.Header.Get("X-KEDA-HTTP-Cold-Start") != "false" {
return fmt.Errorf("expected X-KEDA-HTTP-Cold-Start false, but got %s", resp.Header.Get("X-KEDA-HTTP-Cold-Start"))
}
return nil
})
time.Sleep(100 * time.Millisecond)
select {
case hostAndCount := <-q.ResizedCh:
r.Equal(namespacedName, hostAndCount.Host)
r.Equal(1, hostAndCount.Count)
case <-time.After(2000 * time.Millisecond):
r.Fail("timeout waiting for +1 queue resize")
}
// tell the wait func to proceed
select {
case waiterCh <- struct{}{}:
case <-time.After(5 * time.Second):
r.Fail("timeout producing on waiterCh")
}
select {
case hostAndCount := <-q.ResizedCh:
r.Equal(namespacedName, hostAndCount.Host)
r.Equal(1, hostAndCount.Count)
case <-time.After(2 * time.Second):
r.Fail("timeout waiting for -1 queue resize")
}
// check the queue to make sure all counts are at 0
countsPtr, err := q.Current()
r.NoError(err)
counts := countsPtr.Counts
r.Equal(1, len(counts))
_, foundHost := counts[namespacedName]
r.True(
foundHost,
"couldn't find host %s in the queue",
host,
)
r.Equal(0, counts[namespacedName].Concurrency)
done()
r.Error(g.Wait())
}

View File

@ -21,13 +21,15 @@ type Routing struct {
routingTable routing.Table
probeHandler http.Handler
upstreamHandler http.Handler
tlsEnabled bool
}
func NewRouting(routingTable routing.Table, probeHandler http.Handler, upstreamHandler http.Handler) *Routing {
func NewRouting(routingTable routing.Table, probeHandler http.Handler, upstreamHandler http.Handler, tlsEnabled bool) *Routing {
return &Routing{
routingTable: routingTable,
probeHandler: probeHandler,
upstreamHandler: upstreamHandler,
tlsEnabled: tlsEnabled,
}
}
@ -63,6 +65,14 @@ func (rm *Routing) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
func (rm *Routing) streamFromHTTPSO(httpso *httpv1alpha1.HTTPScaledObject) (*url.URL, error) {
if rm.tlsEnabled {
return url.Parse(fmt.Sprintf(
"https://%s.%s:%d",
httpso.Spec.ScaleTargetRef.Service,
httpso.GetNamespace(),
httpso.Spec.ScaleTargetRef.Port,
))
}
//goland:noinspection HttpUrlsUsage
return url.Parse(fmt.Sprintf(
"http://%s.%s:%d",

View File

@ -23,7 +23,7 @@ var _ = Describe("RoutingMiddleware", func() {
probeHandler.Handle("/probe", emptyHandler)
upstreamHandler.Handle("/upstream", emptyHandler)
rm := NewRouting(routingTable, probeHandler, upstreamHandler)
rm := NewRouting(routingTable, probeHandler, upstreamHandler, false)
Expect(rm).NotTo(BeNil())
Expect(rm.routingTable).To(Equal(routingTable))
Expect(rm.probeHandler).To(Equal(probeHandler))
@ -58,7 +58,7 @@ var _ = Describe("RoutingMiddleware", func() {
upstreamHandler = http.NewServeMux()
probeHandler = http.NewServeMux()
routingTable = routingtest.NewTable()
routingMiddleware = NewRouting(routingTable, probeHandler, upstreamHandler)
routingMiddleware = NewRouting(routingTable, probeHandler, upstreamHandler, false)
w = httptest.NewRecorder()

View File

@ -2,6 +2,7 @@ package main
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"strconv"
@ -48,6 +49,7 @@ func newForwardingHandler(
dialCtxFunc kedanet.DialContextFunc,
waitFunc forwardWaitFunc,
fwdCfg forwardingConfig,
tlsCfg *tls.Config,
) http.Handler {
roundTripper := &http.Transport{
Proxy: http.ProxyFromEnvironment,
@ -58,6 +60,7 @@ func newForwardingHandler(
TLSHandshakeTimeout: fwdCfg.tlsHandshakeTimeout,
ExpectContinueTimeout: fwdCfg.expectContinueTimeout,
ResponseHeaderTimeout: fwdCfg.respHeaderTimeout,
TLSClientConfig: tlsCfg,
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()

View File

@ -2,6 +2,7 @@ package main
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
@ -306,7 +307,9 @@ func newHarness(
waitTimeout: activeEndpointsTimeout,
respHeaderTimeout: time.Second,
},
))
&tls.Config{}),
false,
)
proxySrv, proxySrvURL, err := kedanet.StartTestServer(proxyHdl)
if err != nil {

View File

@ -2,10 +2,14 @@ package main
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"log"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strconv"
"strings"
"testing"
@ -23,6 +27,25 @@ import (
"github.com/kedacore/http-add-on/pkg/util"
)
var TestTLSConfig = tls.Config{}
func init() {
caCert, err := os.ReadFile("../certs/tls.crt")
if err != nil {
log.Fatalf("Error getting tests certs - make sure to run make test to generate them: %v", err)
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
cert, err := tls.LoadX509KeyPair("../certs/tls.crt", "../certs/tls.key")
if err != nil {
log.Fatalf("Error getting tests certs - make sure to run make test to generate them %v", err)
}
TestTLSConfig.RootCAs = caCertPool
TestTLSConfig.Certificates = []tls.Certificate{cert}
}
// the proxy should successfully forward a request to a running server
func TestImmediatelySuccessfulProxy(t *testing.T) {
host := fmt.Sprintf("%s.testing", t.Name())
@ -54,6 +77,7 @@ func TestImmediatelySuccessfulProxy(t *testing.T) {
waitTimeout: timeouts.WorkloadReplicas,
respHeaderTimeout: timeouts.ResponseHeader,
},
&tls.Config{},
)
const path = "/testfwd"
res, req, err := reqAndRes(path)
@ -74,6 +98,57 @@ func TestImmediatelySuccessfulProxy(t *testing.T) {
r.Equal("test response", res.Body.String())
}
func TestImmediatelySuccessfulProxyTLS(t *testing.T) {
host := fmt.Sprintf("%s.testing", t.Name())
r := require.New(t)
originHdl := kedanet.NewTestHTTPHandlerWrapper(
http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(200)
_, err := w.Write([]byte("test response"))
r.NoError(err)
}),
)
srv, originURL, err := kedanet.StartTestServer(originHdl)
r.NoError(err)
defer srv.Close()
originPort, err := strconv.Atoi(originURL.Port())
r.NoError(err)
timeouts := defaultTimeouts()
dialCtxFunc := retryDialContextFunc(timeouts, timeouts.DefaultBackoff())
waitFunc := func(context.Context, string, string) (bool, error) {
return false, nil
}
hdl := newForwardingHandler(
logr.Discard(),
dialCtxFunc,
waitFunc,
forwardingConfig{
waitTimeout: timeouts.WorkloadReplicas,
respHeaderTimeout: timeouts.ResponseHeader,
},
&TestTLSConfig,
)
const path = "/testfwd"
res, req, err := reqAndRes(path)
r.NoError(err)
req = util.RequestWithHTTPSO(req, targetFromURL(
originURL,
originPort,
"testdepl",
"testsvc",
))
req = util.RequestWithStream(req, originURL)
req.Host = host
hdl.ServeHTTP(res, req)
r.Equal("false", res.Header().Get("X-KEDA-HTTP-Cold-Start"), "expected X-KEDA-HTTP-Cold-Start false")
r.Equal(200, res.Code, "expected response code 200")
r.Equal("test response", res.Body.String())
}
// the proxy should wait for a timeout and fail if there is no
// origin to which to connect
func TestWaitFailedConnection(t *testing.T) {
@ -98,6 +173,7 @@ func TestWaitFailedConnection(t *testing.T) {
waitTimeout: timeouts.WorkloadReplicas,
respHeaderTimeout: timeouts.ResponseHeader,
},
&tls.Config{},
)
stream, err := url.Parse("http://0.0.0.0:0")
r.NoError(err)
@ -125,6 +201,57 @@ func TestWaitFailedConnection(t *testing.T) {
r.Equal(502, res.Code, "response code was unexpected")
}
func TestWaitFailedConnectionTLS(t *testing.T) {
const host = "TestWaitFailedConnection.testing"
r := require.New(t)
timeouts := defaultTimeouts()
backoff := timeouts.DefaultBackoff()
backoff.Steps = 2
dialCtxFunc := retryDialContextFunc(
timeouts,
backoff,
)
waitFunc := func(context.Context, string, string) (bool, error) {
return false, nil
}
hdl := newForwardingHandler(
logr.Discard(),
dialCtxFunc,
waitFunc,
forwardingConfig{
waitTimeout: timeouts.WorkloadReplicas,
respHeaderTimeout: timeouts.ResponseHeader,
},
&TestTLSConfig,
)
stream, err := url.Parse("http://0.0.0.0:0")
r.NoError(err)
const path = "/testfwd"
res, req, err := reqAndRes(path)
r.NoError(err)
req = util.RequestWithHTTPSO(req, &httpv1alpha1.HTTPScaledObject{
ObjectMeta: metav1.ObjectMeta{
Namespace: "testns",
},
Spec: httpv1alpha1.HTTPScaledObjectSpec{
ScaleTargetRef: httpv1alpha1.ScaleTargetRef{
Deployment: "nosuchdepl",
Service: "nosuchdepl",
Port: 8081,
},
TargetPendingRequests: ptr.To[int32](1234),
},
})
req = util.RequestWithStream(req, stream)
req.Host = host
hdl.ServeHTTP(res, req)
r.Equal("false", res.Header().Get("X-KEDA-HTTP-Cold-Start"), "expected X-KEDA-HTTP-Cold-Start false")
r.Equal(502, res.Code, "response code was unexpected")
}
// the proxy handler should wait for the wait function until it hits
// a timeout, then it should fail
func TestTimesOutOnWaitFunc(t *testing.T) {
@ -147,6 +274,7 @@ func TestTimesOutOnWaitFunc(t *testing.T) {
waitTimeout: timeouts.WorkloadReplicas,
respHeaderTimeout: timeouts.ResponseHeader,
},
&tls.Config{},
)
stream, err := url.Parse("http://1.1.1.1")
r.NoError(err)
@ -198,6 +326,79 @@ func TestTimesOutOnWaitFunc(t *testing.T) {
r.True(waitFuncCalled, "wait function was not called")
}
func TestTimesOutOnWaitFuncTLS(t *testing.T) {
r := require.New(t)
timeouts := defaultTimeouts()
timeouts.WorkloadReplicas = 25 * time.Millisecond
timeouts.ResponseHeader = 25 * time.Millisecond
dialCtxFunc := retryDialContextFunc(timeouts, timeouts.DefaultBackoff())
waitFunc, waitFuncCalledCh, finishWaitFunc := notifyingFunc()
defer finishWaitFunc()
noSuchHost := fmt.Sprintf("%s.testing", t.Name())
hdl := newForwardingHandler(
logr.Discard(),
dialCtxFunc,
waitFunc,
forwardingConfig{
waitTimeout: timeouts.WorkloadReplicas,
respHeaderTimeout: timeouts.ResponseHeader,
},
&TestTLSConfig,
)
stream, err := url.Parse("http://1.1.1.1")
r.NoError(err)
const path = "/testfwd"
res, req, err := reqAndRes(path)
r.NoError(err)
req = util.RequestWithHTTPSO(req, &httpv1alpha1.HTTPScaledObject{
ObjectMeta: metav1.ObjectMeta{
Namespace: "testns",
},
Spec: httpv1alpha1.HTTPScaledObjectSpec{
ScaleTargetRef: httpv1alpha1.ScaleTargetRef{
Deployment: "nosuchdepl",
Service: "nosuchsvc",
Port: 9091,
},
TargetPendingRequests: ptr.To[int32](1234),
},
})
req = util.RequestWithStream(req, stream)
req.Host = noSuchHost
start := time.Now()
hdl.ServeHTTP(res, req)
elapsed := time.Since(start)
t.Logf("elapsed time was %s", elapsed)
// serving should take at least timeouts.DeploymentReplicas, but no more than
// timeouts.DeploymentReplicas*4
r.GreaterOrEqual(elapsed, timeouts.WorkloadReplicas)
r.LessOrEqual(elapsed, timeouts.WorkloadReplicas*4)
r.Equal(502, res.Code, "response code was unexpected")
// we will always return the X-KEDA-HTTP-Cold-Start header
// when we are able to forward the
// request to the backend but not if we have failed due
// to a timeout from a waitFunc or earlier in the pipeline,
// for example, if we cannot reach the Kubernetes control
// plane.
r.Equal("", res.Header().Get("X-KEDA-HTTP-Cold-Start"), "expected X-KEDA-HTTP-Cold-Start to be empty")
// waitFunc should have been called, even though it timed out
waitFuncCalled := false
select {
case <-waitFuncCalledCh:
waitFuncCalled = true
default:
}
r.True(waitFuncCalled, "wait function was not called")
}
// Test to make sure the proxy handler will wait for the waitFunc to
// complete
func TestWaitsForWaitFunc(t *testing.T) {
@ -228,6 +429,7 @@ func TestWaitsForWaitFunc(t *testing.T) {
waitTimeout: timeouts.WorkloadReplicas,
respHeaderTimeout: timeouts.ResponseHeader,
},
&tls.Config{},
)
const path = "/testfwd"
res, req, err := reqAndRes(path)
@ -265,6 +467,71 @@ func TestWaitsForWaitFunc(t *testing.T) {
)
}
func TestWaitsForWaitFuncTLS(t *testing.T) {
r := require.New(t)
timeouts := defaultTimeouts()
dialCtxFunc := retryDialContextFunc(timeouts, timeouts.DefaultBackoff())
waitFunc, waitFuncCalledCh, finishWaitFunc := notifyingFunc()
const (
noSuchHost = "TestWaitsForWaitFunc.test"
originRespCode = 201
)
testSrv, testSrvURL, err := kedanet.StartTestServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(originRespCode)
}),
)
r.NoError(err)
defer testSrv.Close()
_, originPort, err := splitHostPort(testSrvURL.Host)
r.NoError(err)
hdl := newForwardingHandler(
logr.Discard(),
dialCtxFunc,
waitFunc,
forwardingConfig{
waitTimeout: timeouts.WorkloadReplicas,
respHeaderTimeout: timeouts.ResponseHeader,
},
&TestTLSConfig,
)
const path = "/testfwd"
res, req, err := reqAndRes(path)
r.NoError(err)
req = util.RequestWithHTTPSO(req, targetFromURL(
testSrvURL,
originPort,
"nosuchdepl",
"nosuchsvc",
))
req = util.RequestWithStream(req, testSrvURL)
req.Host = noSuchHost
// make the wait function finish after a short duration
const waitDur = 100 * time.Millisecond
go func() {
time.Sleep(waitDur)
finishWaitFunc()
}()
start := time.Now()
hdl.ServeHTTP(res, req)
elapsed := time.Since(start)
r.NoError(waitForSignal(waitFuncCalledCh, 1*time.Second))
// should take at least waitDur, but no more than waitDur*4
r.GreaterOrEqual(elapsed, waitDur)
r.Less(elapsed, waitDur*4)
r.Equal(
originRespCode,
res.Code,
"response code was unexpected",
)
}
// the proxy should connect to a server, and then time out if
// the server doesn't respond in time
func TestWaitHeaderTimeout(t *testing.T) {
@ -298,6 +565,7 @@ func TestWaitHeaderTimeout(t *testing.T) {
waitTimeout: timeouts.WorkloadReplicas,
respHeaderTimeout: timeouts.ResponseHeader,
},
&tls.Config{},
)
const path = "/testfwd"
res, req, err := reqAndRes(path)
@ -324,6 +592,65 @@ func TestWaitHeaderTimeout(t *testing.T) {
close(originHdlCh)
}
func TestWaitHeaderTimeoutTLS(t *testing.T) {
r := require.New(t)
// the origin will wait for this channel to receive or close before it sends any data back to the
// proxy
originHdlCh := make(chan struct{})
originHdl := kedanet.NewTestHTTPHandlerWrapper(
http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
<-originHdlCh
w.WriteHeader(200)
_, err := w.Write([]byte("test response"))
r.NoError(err)
}),
)
srv, originURL, err := kedanet.StartTestServer(originHdl)
r.NoError(err)
defer srv.Close()
timeouts := defaultTimeouts()
dialCtxFunc := retryDialContextFunc(timeouts, timeouts.DefaultBackoff())
waitFunc := func(context.Context, string, string) (bool, error) {
return false, nil
}
hdl := newForwardingHandler(
logr.Discard(),
dialCtxFunc,
waitFunc,
forwardingConfig{
waitTimeout: timeouts.WorkloadReplicas,
respHeaderTimeout: timeouts.ResponseHeader,
},
&TestTLSConfig,
)
const path = "/testfwd"
res, req, err := reqAndRes(path)
r.NoError(err)
req = util.RequestWithHTTPSO(req, &httpv1alpha1.HTTPScaledObject{
ObjectMeta: metav1.ObjectMeta{
Namespace: "testns",
},
Spec: httpv1alpha1.HTTPScaledObjectSpec{
ScaleTargetRef: httpv1alpha1.ScaleTargetRef{
Deployment: "nosuchdepl",
Service: "testsvc",
Port: 9094,
},
TargetPendingRequests: ptr.To[int32](1234),
},
})
req = util.RequestWithStream(req, originURL)
req.Host = originURL.Host
hdl.ServeHTTP(res, req)
r.Equal("false", res.Header().Get("X-KEDA-HTTP-Cold-Start"), "expected X-KEDA-HTTP-Cold-Start false")
r.Equal(502, res.Code, "response code was unexpected")
close(originHdlCh)
}
func waitForSignal(sig <-chan struct{}, waitDur time.Duration) error {
tmr := time.NewTimer(waitDur)
defer tmr.Stop()

View File

@ -7,7 +7,7 @@ import (
"github.com/kedacore/http-add-on/pkg/util"
)
func ServeContext(ctx context.Context, addr string, hdl http.Handler) error {
func ServeContext(ctx context.Context, addr string, hdl http.Handler, tlsEnabled bool, tlsConfig map[string]string) error {
srv := &http.Server{
Handler: hdl,
Addr: addr,
@ -22,5 +22,9 @@ func ServeContext(ctx context.Context, addr string, hdl http.Handler) error {
}
}()
if tlsEnabled {
return srv.ListenAndServeTLS(tlsConfig["certificatePath"], tlsConfig["keyPath"])
}
return srv.ListenAndServe()
}

View File

@ -29,7 +29,35 @@ func TestServeContext(t *testing.T) {
done()
}()
start := time.Now()
err := ServeContext(ctx, addr, hdl)
err := ServeContext(ctx, addr, hdl, false, map[string]string{})
elapsed := time.Since(start)
r.Error(err)
r.True(errors.Is(err, http.ErrServerClosed), "error is not a http.ErrServerClosed (%w)", err)
r.Greater(elapsed, cancelDur)
r.Less(elapsed, cancelDur*4)
}
func TestServeContextWithTLS(t *testing.T) {
r := require.New(t)
ctx, done := context.WithCancel(
context.Background(),
)
hdl := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("foo", "bar")
_, err := w.Write([]byte("hello world"))
if err != nil {
t.Fatalf("error writing message to client from handler")
}
})
addr := "localhost:1234"
const cancelDur = 500 * time.Millisecond
go func() {
time.Sleep(cancelDur)
done()
}()
start := time.Now()
err := ServeContext(ctx, addr, hdl, true, map[string]string{"certificatePath": "../../certs/tls.crt", "keyPath": "../../certs/tls.key"})
elapsed := time.Since(start)
r.Error(err)

View File

@ -0,0 +1,201 @@
//go:build e2e
// +build e2e
package interceptor_tls_test
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
. "github.com/kedacore/http-add-on/tests/helper"
)
const (
testName = "interceptor-tls-test"
)
var (
testNamespace = fmt.Sprintf("%s-ns", testName)
deploymentName = fmt.Sprintf("%s-deployment", testName)
serviceName = fmt.Sprintf("%s-service", testName)
clientName = fmt.Sprintf("%s-client", testName)
httpScaledObjectName = fmt.Sprintf("%s-http-so", testName)
host = testName
minReplicaCount = 0
maxReplicaCount = 1
)
type templateData struct {
TestNamespace string
DeploymentName string
ServiceName string
ClientName string
HTTPScaledObjectName string
Host string
MinReplicas int
MaxReplicas int
}
const (
serviceTemplate = `
apiVersion: v1
kind: Service
metadata:
name: {{.ServiceName}}
namespace: {{.TestNamespace}}
labels:
app: {{.DeploymentName}}
spec:
ports:
- port: 8080
targetPort: http
protocol: TCP
name: http
- port: 8443
targetPort: https
protocol: TCP
name: https
selector:
app: {{.DeploymentName}}
`
deploymentTemplate = `
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{.DeploymentName}}
namespace: {{.TestNamespace}}
labels:
app: {{.DeploymentName}}
spec:
replicas: 0
selector:
matchLabels:
app: {{.DeploymentName}}
template:
metadata:
labels:
app: {{.DeploymentName}}
spec:
containers:
- name: {{.DeploymentName}}
image: registry.k8s.io/e2e-test-images/agnhost:2.45
args:
- netexec
- --http-port
- "8443"
- --tls-cert-file
- /certs/tls.crt
- --tls-private-key-file
- /certs/tls.key
ports:
- name: http
containerPort: 8080
protocol: TCP
- name: https
containerPort: 8443
protocol: TCP
volumeMounts:
- readOnly: true
mountPath: "/certs"
name: certs
readinessProbe:
httpGet:
path: /
port: https
scheme: HTTPS
volumes:
- name: certs
secret:
secretName: test-tls
`
httpScaledObjectTemplate = `
kind: HTTPScaledObject
apiVersion: http.keda.sh/v1alpha1
metadata:
name: {{.HTTPScaledObjectName}}
namespace: {{.TestNamespace}}
spec:
hosts:
- {{.Host}}
targetPendingRequests: 100
scaledownPeriod: 10
scaleTargetRef:
deployment: {{.DeploymentName}}
service: {{.ServiceName}}
port: 8443
replicas:
min: {{ .MinReplicas }}
max: {{ .MaxReplicas }}
`
clientTemplate = `
apiVersion: v1
kind: Pod
metadata:
name: {{.ClientName}}
namespace: {{.TestNamespace}}
spec:
containers:
- name: {{.ClientName}}
image: curlimages/curl
command:
- sh
- -c
- "exec tail -f /dev/null"`
)
func TestInterceptorTLS(t *testing.T) {
// setup
t.Log("--- setting up ---")
// create kubernetes resources
kc := GetKubernetesClient(t)
data, templates := getTemplateData()
CreateKubernetesResources(t, kc, testNamespace, data, templates)
// setup certs
_, err := ExecuteCommand(fmt.Sprintf("kubectl -n %s create secret tls test-tls --cert ../../../certs/tls.crt --key ../../../certs/tls.key", testNamespace))
require.NoErrorf(t, err, "could not create tls cert secret in %s namespace - %s", testNamespace, err)
// wait for test pod to start
assert.True(t, WaitForAllPodRunningInNamespace(t, kc, testNamespace, 10, 2),
"test client count should be available after 20 seconds")
// send test request and validate response body
sendRequest(t)
// cleanup
DeleteKubernetesResources(t, testNamespace, data, templates)
}
func sendRequest(t *testing.T) {
t.Log("--- sending request ---")
stdout, _, err := ExecCommandOnSpecificPod(t, clientName, testNamespace, fmt.Sprintf("curl -k -H 'Host: %s' https://keda-http-add-on-interceptor-proxy.keda:8443/echo?msg=tls_test", host))
require.NoErrorf(t, err, "could not run command on test client pod - %s", err)
assert.Equal(t, "tls_test", stdout, fmt.Sprintf("incorrect response body from test request: expected %s, got %s", "tls_test", stdout))
}
func getTemplateData() (templateData, []Template) {
return templateData{
TestNamespace: testNamespace,
DeploymentName: deploymentName,
ServiceName: serviceName,
ClientName: clientName,
HTTPScaledObjectName: httpScaledObjectName,
Host: host,
MinReplicas: minReplicaCount,
MaxReplicas: maxReplicaCount,
}, []Template{
{Name: "deploymentTemplate", Config: deploymentTemplate},
{Name: "serviceNameTemplate", Config: serviceTemplate},
{Name: "clientTemplate", Config: clientTemplate},
{Name: "httpScaledObjectTemplate", Config: httpScaledObjectTemplate},
}
}

View File

@ -47,3 +47,10 @@ func TestRemoveOpentelemetryComponents(t *testing.T) {
require.NoErrorf(t, err, "cannot uninstall opentelemetry-collector - %s", err)
DeleteNamespace(t, OpentelemetryNamespace)
}
func TestCleanUpCerts(t *testing.T) {
out, err := ExecuteCommandWithDir("make clean-test-certs", "../..")
require.NoErrorf(t, err, "error cleaning up test certs - %s", err)
t.Log(string(out))
t.Log("test certificates successfully cleaned up")
}

View File

@ -172,6 +172,16 @@ func TestSetupKEDA(t *testing.T) {
"replica count should be 1 after 3 minutes")
}
func TestSetupTLSConfiguration(t *testing.T) {
out, err := ExecuteCommandWithDir("make test-certs", "../..")
require.NoErrorf(t, err, "error generating test certs - %s", err)
t.Log(string(out))
t.Log("test certificates successfully generated")
_, err = ExecuteCommand("kubectl -n keda create secret tls keda-tls --cert ../../certs/tls.crt --key ../../certs/tls.key")
require.NoErrorf(t, err, "could not create tls cert secret in keda namespace - %s", err)
}
func TestDeployKEDAHttpAddOn(t *testing.T) {
out, err := ExecuteCommandWithDir("make deploy", "../..")
require.NoErrorf(t, err, "error deploying KEDA Http Add-on - %s", err)
@ -205,7 +215,6 @@ func TestSetupOpentelemetryComponents(t *testing.T) {
CreateNamespace(t, KubeClient, OpentelemetryNamespace)
_, err = ExecuteCommand(fmt.Sprintf("helm upgrade --install opentelemetry-collector open-telemetry/opentelemetry-collector -f %s --namespace %s", otlpTempFileName, OpentelemetryNamespace))
require.NoErrorf(t, err, "cannot install opentelemetry - %s", err)
_, err = ExecuteCommand(fmt.Sprintf("kubectl apply -f %s -n %s", otlpServiceTempFileName, OpentelemetryNamespace))