webhook/pkg/server/server.go

// Package server is used to create and run the webhook server
package server

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"github.com/rancher/dynamiclistener"
	"github.com/rancher/dynamiclistener/server"
	"github.com/rancher/webhook/pkg/admission"
	"github.com/rancher/webhook/pkg/clients"
	"github.com/rancher/webhook/pkg/health"
	admissionregistration "github.com/rancher/wrangler/v3/pkg/generated/controllers/admissionregistration.k8s.io/v1"
	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/admissionregistration/v1"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/rest"
)

const (
	serviceName             = "rancher-webhook"
	namespace               = "cattle-system"
	tlsName                 = "rancher-webhook.cattle-system.svc"
	certName                = "cattle-webhook-tls"
	caName                  = "cattle-webhook-ca"
	validationPath          = "/v1/webhook/validation"
	mutationPath            = "/v1/webhook/mutation"
	clientPort              = int32(443)
	webhookHTTPPort         = 0 // value of 0 indicates we do not want to use http.
	defaultWebhookHTTPSPort = 9443
	webhookPortEnvKey       = "CATTLE_PORT"
	webhookURLEnvKey        = "CATTLE_WEBHOOK_URL"
	allowedCNsEnv           = "ALLOWED_CNS"
)

var caFile = filepath.Join(os.TempDir(), "k8s-webhook-server", "client-ca", "ca.crt")

// tlsOpt option function applied to all webhook servers.
var tlsOpt = func(config *tls.Config) {
	config.MinVersion = tls.VersionTLS12
	config.CipherSuites = []uint16{
		tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
		tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
		tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
		tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
	}
	config.ClientAuth = tls.RequestClientCert
}

// ListenAndServe starts the webhook server.
func ListenAndServe(ctx context.Context, cfg *rest.Config, mcmEnabled bool) error {
	clients, err := clients.New(ctx, cfg, mcmEnabled)
	if err != nil {
		return fmt.Errorf("failed to create a new client: %w", err)
	}

	if err = setCertificateExpirationDays(); err != nil {
		// If this error occurs, certificate creation will still work. However, our override will likely not have worked.
		// This will not affect functionality of the webhook, but users may have to perform the workaround:
		// https://github.com/rancher/docs/issues/3637
		logrus.Infof("[ListenAndServe] could not set certificate expiration days via environment variable: %v", err)
	}

	validators, err := Validation(clients)
	if err != nil {
		return err
	}

	mutators, err := Mutation(clients)
	if err != nil {
		return err
	}

	if err = listenAndServe(ctx, clients, validators, mutators); err != nil {
		return err
	}

	if err = clients.Start(ctx); err != nil {
		return fmt.Errorf("failed to start client: %w", err)
	}

	return nil
}

// By default, dynamiclistener sets newly signed certificates to expire after 365 days. Since the
// self-signed certificate for webhook does not need to be rotated, we increase expiration time
// beyond relevance. In this case, that's 3650 days (10 years).
func setCertificateExpirationDays() error {
	certExpirationDaysKey := "CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS"
	if os.Getenv(certExpirationDaysKey) == "" {
		return os.Setenv(certExpirationDaysKey, "3650")
	}
	return nil
}

func listenAndServe(ctx context.Context, clients *clients.Clients, validators []admission.ValidatingAdmissionHandler, mutators []admission.MutatingAdmissionHandler) (rErr error) {
	router := mux.NewRouter()
	errChecker := health.NewErrorChecker("Config Applied")
	health.RegisterHealthCheckers(router, errChecker)
	router.Use(certAuth())

	logrus.Debug("Creating Webhook routes")
	for _, webhook := range validators {
		route := router.HandleFunc(admission.Path(validationPath, webhook), admission.NewValidatingHandlerFunc(webhook))
		path, _ := route.GetPathTemplate()
		logrus.Debugf("creating route: %s", path)
	}
	for _, webhook := range mutators {
		route := router.HandleFunc(admission.Path(mutationPath, webhook), admission.NewMutatingHandlerFunc(webhook))
		path, _ := route.GetPathTemplate()
		logrus.Debugf("creating route: %s", path)
	}

	handler := &secretHandler{
		validators:           validators,
		mutators:             mutators,
		errChecker:           errChecker,
		validatingController: clients.Admission.ValidatingWebhookConfiguration(),
		mutatingController:   clients.Admission.MutatingWebhookConfiguration(),
	}
	clients.Core.Secret().OnChange(ctx, "secrets", handler.sync)

	defer func() {
		if rErr != nil {
			return
		}
		rErr = clients.Start(ctx)
	}()

	tlsConfig := &tls.Config{}
	tlsOpt(tlsConfig)
	webhookHTTPSPort := defaultWebhookHTTPSPort
	if portStr := os.Getenv(webhookPortEnvKey); portStr != "" {
		var err error
		webhookHTTPSPort, err = strconv.Atoi(portStr)
		if err != nil {
			return fmt.Errorf("failed to decode webhook port value '%s': %w", portStr, err)
		}
	}
	return server.ListenAndServe(ctx, webhookHTTPSPort, webhookHTTPPort, router, &server.ListenOpts{
		Secrets:       clients.Core.Secret(),
		CertNamespace: namespace,
		CertName:      certName,
		CAName:        caName,
		TLSListenerConfig: dynamiclistener.Config{
			SANs: []string{
				tlsName,
			},
			FilterCN:  dynamiclistener.OnlyAllow(tlsName),
			TLSConfig: tlsConfig,
		},
		DisplayServerLogs:       true,
		IgnoreTLSHandshakeError: true,
	})
}

type secretHandler struct {
	validators           []admission.ValidatingAdmissionHandler
	mutators             []admission.MutatingAdmissionHandler
	errChecker           *health.ErrorChecker
	validatingController admissionregistration.ValidatingWebhookConfigurationClient
	mutatingController   admissionregistration.MutatingWebhookConfigurationClient
}

// sync updates the validating admission configuration whenever the TLS cert changes.
func (s *secretHandler) sync(_ string, secret *corev1.Secret) (*corev1.Secret, error) {
	if secret == nil || secret.Name != caName || secret.Namespace != namespace || len(secret.Data[corev1.TLSCertKey]) == 0 {
		return nil, nil
	}

	logrus.Info("Sleeping for 15 seconds then applying webhook config")
	// Sleep here to make sure server is listening and all caches are primed
	time.Sleep(15 * time.Second)

	validationClientConfig := v1.WebhookClientConfig{
		Service: &v1.ServiceReference{
			Namespace: namespace,
			Name:      serviceName,
			Path:      admission.Ptr(validationPath),
			Port:      admission.Ptr(clientPort),
		},
		CABundle: secret.Data[corev1.TLSCertKey],
	}

	mutationClientConfig := v1.WebhookClientConfig{
		Service: &v1.ServiceReference{
			Namespace: namespace,
			Name:      serviceName,
			Path:      admission.Ptr(mutationPath),
			Port:      admission.Ptr(clientPort),
		},
		CABundle: secret.Data[corev1.TLSCertKey],
	}
	if devURL, ok := os.LookupEnv(webhookURLEnvKey); ok {
		validationURL := devURL + validationPath
		mutationURL := devURL + mutationPath
		validationClientConfig = v1.WebhookClientConfig{
			URL: &validationURL,
		}
		mutationClientConfig = v1.WebhookClientConfig{
			URL: &mutationURL,
		}
	}
	validatingWebhooks := make([]v1.ValidatingWebhook, 0, len(s.validators))
	for _, webhook := range s.validators {
		validatingWebhooks = append(validatingWebhooks, webhook.ValidatingWebhook(validationClientConfig)...)
	}
	mutatingWebhooks := make([]v1.MutatingWebhook, 0, len(s.mutators))
	for _, webhook := range s.mutators {
		mutatingWebhooks = append(mutatingWebhooks, webhook.MutatingWebhook(mutationClientConfig)...)
	}
	validatingConfig := &v1.ValidatingWebhookConfiguration{
		ObjectMeta: metav1.ObjectMeta{
			Name: "rancher.cattle.io",
		},
		Webhooks: validatingWebhooks,
	}
	mutatingConfig := &v1.MutatingWebhookConfiguration{
		ObjectMeta: metav1.ObjectMeta{
			Name: "rancher.cattle.io",
		},
		Webhooks: mutatingWebhooks,
	}
	err := s.ensureWebhookConfiguration(validatingConfig, mutatingConfig)
	if err != nil {
		logrus.Errorf("Failed to ensure configuration: %s", err.Error())
	}

	s.errChecker.Store(err)
	return secret, err

}

// ensureWebhookConfiguration creates or updates the current validating and mutating webhook configuration to have the desired webhook.
func (s *secretHandler) ensureWebhookConfiguration(validatingConfig *v1.ValidatingWebhookConfiguration, mutatingConfig *v1.MutatingWebhookConfiguration) error {

	currValidating, err := s.validatingController.Get(validatingConfig.Name, metav1.GetOptions{})
	if apierrors.IsNotFound(err) {
		_, err = s.validatingController.Create(validatingConfig)
		if err != nil {
			return fmt.Errorf("failed to create validating configuration: %w", err)
		}
	} else if err != nil {
		return fmt.Errorf("failed to get validating configuration: %w", err)
	} else {
		currValidating.Webhooks = validatingConfig.Webhooks
		_, err = s.validatingController.Update(currValidating)
		if err != nil {
			return fmt.Errorf("failed to update validating configuration: %w", err)
		}
	}

	currMutation, err := s.mutatingController.Get(mutatingConfig.Name, metav1.GetOptions{})
	if apierrors.IsNotFound(err) {
		_, err = s.mutatingController.Create(mutatingConfig)
		if err != nil {
			return fmt.Errorf("failed to create mutating configuration: %w", err)
		}
	} else if err != nil {
		return fmt.Errorf("failed to get mutating configuration: %w", err)
	} else {
		currMutation.Webhooks = mutatingConfig.Webhooks
		_, err = s.mutatingController.Update(currMutation)
		if err != nil {
			return fmt.Errorf("failed to update mutating configuration: %w", err)
		}
	}

	return nil
}

// certAuth returns a middleware for cert-based authentication.
// This is done as a middleware instead of using tls.RequireAndVerifyClientCert because an exception
// needs to be made for the unauthenticated /healthz endpoint.
func certAuth() func(next http.Handler) http.Handler {
	opts := getVerifyOptions()
	allowedCNs := getAllowedCNs()
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			logrus.Tracef("running cert check middleware for request %s", r.URL.Path)
			if opts == nil {
				next.ServeHTTP(w, r)
				return
			}
			if r.URL.Path == "/healthz" { // apiserver does not present client cert for health checks
				next.ServeHTTP(w, r)
				return
			}
			if len(r.TLS.PeerCertificates) == 0 {
				logrus.Warn("client did not present certificates")
				http.Error(w, "could not verify client certificates", http.StatusUnauthorized)
				return
			}
			for _, cert := range r.TLS.PeerCertificates[1:] {
				opts.Intermediates.AddCert(cert)
			}
			_, err := r.TLS.PeerCertificates[0].Verify(*opts)
			if err != nil {
				logrus.Warnf("could not verify client certificates: %v", err)
				http.Error(w, "could not verify client certificates", http.StatusUnauthorized)
				return
			}
			if len(allowedCNs) == 0 {
				next.ServeHTTP(w, r)
				return
			}
			requestCN := r.TLS.PeerCertificates[0].Subject.CommonName
			found := false
			for _, allowed := range allowedCNs {
				if allowed == requestCN {
					found = true
					break
				}
			}
			if !found {
				logrus.Warnf("could not find common name %s in allowed list", requestCN)
				http.Error(w, "common name is not allowed", http.StatusUnauthorized)
				return
			}
			next.ServeHTTP(w, r)
		})
	}
}

func getVerifyOptions() *x509.VerifyOptions {
	caCert, err := os.ReadFile(caFile)
	if err != nil {
		logrus.Infof("could not read client CA file at %s, incoming requests will not be authenticated", caFile)
		return nil
	}
	caCertPool := x509.NewCertPool()
	caCertPool.AppendCertsFromPEM(caCert)
	opts := x509.VerifyOptions{
		Roots:         caCertPool,
		Intermediates: x509.NewCertPool(),
		KeyUsages:     []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
	}
	return &opts
}

func getAllowedCNs() []string {
	allowedCNString := os.Getenv(allowedCNsEnv)
	if len(allowedCNString) == 0 {
		return nil
	}
	return strings.Split(allowedCNString, ",")
}