wire up a means to dynamically reload ca bundles for kube-apiserver

Kubernetes-commit: 6beb96261e29754f2b7d0e44829eb6d15422cebf

pkg/authentication/authenticatorfactory/delegating.go

@@ -30,7 +30,6 @@ import (
 	unionauth "k8s.io/apiserver/pkg/authentication/request/union"
 	"k8s.io/apiserver/pkg/authentication/request/websocket"
 	"k8s.io/apiserver/pkg/authentication/request/x509"
-	x509request "k8s.io/apiserver/pkg/authentication/request/x509"
 	"k8s.io/apiserver/pkg/authentication/token/cache"
 	webhooktoken "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
 	authenticationclient "k8s.io/client-go/kubernetes/typed/authentication/v1beta1"
@@ -47,9 +46,10 @@ type DelegatingAuthenticatorConfig struct {
 	// CacheTTL is the length of time that a token authentication answer will be cached.
 	CacheTTL time.Duration
 
-	// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
+	// CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
 	// Generally this is the CA bundle file used to authenticate client certificates
-	ClientVerifyOptionFn x509request.VerifyOptionFunc
+	// If this is nil, then mTLS will not be used.
+	ClientCertificateCAContentProvider CAContentProvider
 
 	APIAudiences authenticator.Audiences
 
@@ -64,7 +64,7 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
 	// Add the front proxy authenticator if requested
 	if c.RequestHeaderConfig != nil {
 		requestHeaderAuthenticator := headerrequest.NewDynamicVerifyOptionsSecure(
-			c.RequestHeaderConfig.VerifyOptionFn,
+			c.RequestHeaderConfig.CAContentProvider.VerifyOptions,
 			c.RequestHeaderConfig.AllowedClientNames,
 			c.RequestHeaderConfig.UsernameHeaders,
 			c.RequestHeaderConfig.GroupHeaders,
@@ -74,8 +74,8 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
 	}
 
 	// x509 client cert auth
-	if c.ClientVerifyOptionFn != nil {
-		authenticators = append(authenticators, x509.NewDynamic(c.ClientVerifyOptionFn, x509.CommonNameUserConversion))
+	if c.ClientCertificateCAContentProvider != nil {
+		authenticators = append(authenticators, x509.NewDynamic(c.ClientCertificateCAContentProvider.VerifyOptions, x509.CommonNameUserConversion))
 	}
 
 	if c.TokenAccessReviewClient != nil {

pkg/authentication/authenticatorfactory/requestheader.go

@@ -17,8 +17,9 @@ limitations under the License.
 package authenticatorfactory
 
 import (
+	"crypto/x509"
+
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
-	x509request "k8s.io/apiserver/pkg/authentication/request/x509"
 )
 
 type RequestHeaderConfig struct {
@@ -29,9 +30,19 @@ type RequestHeaderConfig struct {
 	// ExtraHeaderPrefixes are the head prefixes to check (case-insentively) for filling in
 	// the user.Info.Extra.  All values of all matching headers will be added.
 	ExtraHeaderPrefixes headerrequest.StringSliceProvider
-	// VerifyOptionFn are the options for verifying incoming connections using mTLS.  Generally this points to CA bundle file which is used verify the identity of the front proxy.
-	// It may produce different options at will.
-	VerifyOptionFn x509request.VerifyOptionFunc
+	// CAContentProvider the options for verifying incoming connections using mTLS.  Generally this points to CA bundle file which is used verify the identity of the front proxy.
+	//	It may produce different options at will.
+	CAContentProvider CAContentProvider
 	// AllowedClientNames is a list of common names that may be presented by the authenticating front proxy.  Empty means: accept any.
 	AllowedClientNames headerrequest.StringSliceProvider
 }
+
+// CAContentProvider provides ca bundle byte content
+type CAContentProvider interface {
+	// Name is just an identifier
+	Name() string
+	// CurrentCABundleContent provides ca bundle byte content
+	CurrentCABundleContent() []byte
+	// VerifyOptions provides VerifyOptions for authenticators
+	VerifyOptions() x509.VerifyOptions
+}

pkg/server/config.go

@@ -345,21 +345,19 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de
 	}
 }
 
-func (c *AuthenticationInfo) ApplyClientCert(clientCAFile string, servingInfo *SecureServingInfo) error {
-	if servingInfo != nil {
-		if len(clientCAFile) > 0 {
-			clientCAProvider, err := dynamiccertificates.NewStaticCAContentFromFile(clientCAFile)
-			if err != nil {
-				return fmt.Errorf("unable to load client CA file: %v", err)
-			}
-			if servingInfo.ClientCA == nil {
-				servingInfo.ClientCA = clientCAProvider
-			} else {
-				servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCAProvider)
-			}
-		}
+func (c *AuthenticationInfo) ApplyClientCert(clientCA dynamiccertificates.CAContentProvider, servingInfo *SecureServingInfo) error {
+	if servingInfo == nil {
+		return nil
+	}
+	if clientCA == nil {
+		return nil
+	}
+	if servingInfo.ClientCA == nil {
+		servingInfo.ClientCA = clientCA
+		return nil
 	}
 
+	servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCA)
 	return nil
 }
 

pkg/server/dynamiccertificates/client_ca.go

@@ -18,6 +18,7 @@ package dynamiccertificates
 
 import (
 	"bytes"
+	"crypto/x509"
 )
 
 // CAContentProvider provides ca bundle byte content
@@ -27,6 +28,8 @@ type CAContentProvider interface {
 	// CurrentCABundleContent provides ca bundle byte content.  Errors can be contained to the controllers initializing
 	// the value.  By the time you get here, you should always be returning a value that won't fail.
 	CurrentCABundleContent() []byte
+	// VerifyOptions provides VerifyOptions for authenticators
+	VerifyOptions() x509.VerifyOptions
 }
 
 // dynamicCertificateContent holds the content that overrides the baseTLSConfig

pkg/server/dynamiccertificates/dynamicfile_content.go

@@ -17,10 +17,10 @@ limitations under the License.
 package dynamiccertificates
 
 import (
+	"bytes"
 	"crypto/x509"
 	"fmt"
 	"io/ioutil"
-	"reflect"
 	"sync/atomic"
 	"time"
 
@@ -32,10 +32,30 @@ import (
 	"k8s.io/klog"
 )
 
-type CAListener interface {
+// FileRefreshDuration is exposed so that integration tests can crank up the reload speed.
+var FileRefreshDuration = 1 * time.Minute
+
+// Listener is an interface to use to notify interested parties of a change.
+type Listener interface {
+	// Enqueue should be called when an input may have changed
 	Enqueue()
 }
 
+// Notifier is a way to add listeners
+type Notifier interface {
+	// AddListener is adds a listener to be notified of potential input changes
+	AddListener(listener Listener)
+}
+
+// ControllerRunner is a generic interface for starting a controller
+type ControllerRunner interface {
+	// RunOnce runs the sync loop a single time.  This useful for synchronous priming
+	RunOnce() error
+
+	// Run should be called a go .Run
+	Run(workers int, stopCh <-chan struct{})
+}
+
 // DynamicFileCAContent provies a CAContentProvider that can dynamically react to new file content
 // It also fulfills the authenticator interface to provide verifyoptions
 type DynamicFileCAContent struct {
@@ -47,18 +67,22 @@ type DynamicFileCAContent struct {
 	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
 	caBundle atomic.Value
 
-	listeners []CAListener
+	listeners []Listener
 
 	// queue only ever has one item, but it has nice error handling backoff/retry semantics
 	queue workqueue.RateLimitingInterface
 }
 
+var _ Notifier = &DynamicFileCAContent{}
+var _ CAContentProvider = &DynamicFileCAContent{}
+var _ ControllerRunner = &DynamicFileCAContent{}
+
 type caBundleAndVerifier struct {
 	caBundle      []byte
 	verifyOptions x509.VerifyOptions
 }
 
-// NewStaticCAContentFromFile returns a CAContentProvider based on a filename
+// NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content
 func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) {
 	if len(filename) == 0 {
 		return nil, fmt.Errorf("missing filename for ca bundle")
@@ -78,7 +102,7 @@ func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAConten
 }
 
 // AddListener adds a listener to be notified when the CA content changes.
-func (c *DynamicFileCAContent) AddListener(listener CAListener) {
+func (c *DynamicFileCAContent) AddListener(listener Listener) {
 	c.listeners = append(c.listeners, listener)
 }
 
@@ -93,8 +117,7 @@ func (c *DynamicFileCAContent) loadCABundle() error {
 	}
 
 	// check to see if we have a change. If the values are the same, do nothing.
-	existing, ok := c.caBundle.Load().(*caBundleAndVerifier)
-	if ok && existing != nil && reflect.DeepEqual(existing.caBundle, caBundle) {
+	if !c.hasCAChanged(caBundle) {
 		return nil
 	}
 
@@ -111,6 +134,30 @@ func (c *DynamicFileCAContent) loadCABundle() error {
 	return nil
 }
 
+// hasCAChanged returns true if the caBundle is different than the current.
+func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool {
+	uncastExisting := c.caBundle.Load()
+	if uncastExisting == nil {
+		return true
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := uncastExisting.(*caBundleAndVerifier)
+	if !ok {
+		return true
+	}
+	if !bytes.Equal(existing.caBundle, caBundle) {
+		return true
+	}
+
+	return false
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicFileCAContent) RunOnce() error {
+	return c.loadCABundle()
+}
+
 // Run starts the kube-apiserver and blocks until stopCh is closed.
 func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
@@ -123,7 +170,7 @@ func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) {
 	go wait.Until(c.runWorker, time.Second, stopCh)
 
 	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
-	_ = wait.PollImmediateUntil(1*time.Minute, func() (bool, error) {
+	_ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
 		c.queue.Add(workItemKey)
 		return false, nil
 	}, stopCh)
@@ -164,11 +211,12 @@ func (c *DynamicFileCAContent) Name() string {
 
 // CurrentCABundleContent provides ca bundle byte content
 func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) {
-	return c.caBundle.Load().(caBundleAndVerifier).caBundle
+	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
 }
 
+// VerifyOptions provides verifyoptions compatible with authenticators
 func (c *DynamicFileCAContent) VerifyOptions() x509.VerifyOptions {
-	return c.caBundle.Load().(caBundleAndVerifier).verifyOptions
+	return c.caBundle.Load().(*caBundleAndVerifier).verifyOptions
 }
 
 // newVerifyOptions creates a new verification func from a file.  It reads the content and then fails.

pkg/server/dynamiccertificates/static_content.go

@@ -18,15 +18,18 @@ package dynamiccertificates
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"io/ioutil"
 )
 
 type staticCAContent struct {
 	name     string
-	caBundle []byte
+	caBundle *caBundleAndVerifier
 }
 
+var _ CAContentProvider = &staticCAContent{}
+
 // NewStaticCAContentFromFile returns a CAContentProvider based on a filename
 func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) {
 	if len(filename) == 0 {
@@ -37,15 +40,20 @@ func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) {
 	if err != nil {
 		return nil, err
 	}
-	return NewStaticCAContent(filename, caBundle), nil
+	return NewStaticCAContent(filename, caBundle)
 }
 
 // NewStaticCAContent returns a CAContentProvider that always returns the same value
-func NewStaticCAContent(name string, caBundle []byte) CAContentProvider {
+func NewStaticCAContent(name string, caBundle []byte) (CAContentProvider, error) {
+	caBundleAndVerifier, err := newCABundleAndVerifier(name, caBundle)
+	if err != nil {
+		return nil, err
+	}
+
 	return &staticCAContent{
 		name:     name,
-		caBundle: caBundle,
-	}
+		caBundle: caBundleAndVerifier,
+	}, nil
 }
 
 // Name is just an identifier
@@ -55,7 +63,11 @@ func (c *staticCAContent) Name() string {
 
 // CurrentCABundleContent provides ca bundle byte content
 func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
-	return c.caBundle
+	return c.caBundle.caBundle
+}
+
+func (c *staticCAContent) VerifyOptions() x509.VerifyOptions {
+	return c.caBundle.verifyOptions
 }
 
 type staticCertKeyContent struct {

pkg/server/dynamiccertificates/tlsconfig.go

@@ -60,6 +60,8 @@ type DynamicServingCertificateController struct {
 	eventRecorder events.EventRecorder
 }
 
+var _ Listener = &DynamicServingCertificateController{}
+
 // NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date.
 func NewDynamicServingCertificateController(
 	baseTLSConfig tls.Config,

pkg/server/dynamiccertificates/tlsconfig_test.go

@@ -89,7 +89,7 @@ func TestNewStaticCertKeyContent(t *testing.T) {
 	}{
 		{
 			name:        "filled",
-			clientCA:    NewStaticCAContent("test-ca", []byte("content-1")),
+			clientCA:    &staticCAContent{name: "test-ca", caBundle: &caBundleAndVerifier{caBundle: []byte("content-1")}},
 			servingCert: testCertProvider,
 			sniCerts:    []SNICertKeyContentProvider{testCertProvider},
 			expected: &dynamicCertificateContent{
@@ -101,7 +101,7 @@ func TestNewStaticCertKeyContent(t *testing.T) {
 		},
 		{
 			name:        "missingCA",
-			clientCA:    NewStaticCAContent("test-ca", []byte("")),
+			clientCA:    &staticCAContent{name: "test-ca", caBundle: &caBundleAndVerifier{caBundle: []byte("")}},
 			expected:    nil,
 			expectedErr: `not loading an empty client ca bundle from "test-ca"`,
 		},

pkg/server/dynamiccertificates/union_content.go

@@ -18,11 +18,18 @@ package dynamiccertificates
 
 import (
 	"bytes"
+	"crypto/x509"
 	"strings"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 )
 
 type unionCAContent []CAContentProvider
 
+var _ Notifier = &unionCAContent{}
+var _ CAContentProvider = &unionCAContent{}
+var _ ControllerRunner = &unionCAContent{}
+
 // NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders
 func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider {
 	return unionCAContent(caContentProviders)
@@ -46,3 +53,48 @@ func (c unionCAContent) CurrentCABundleContent() []byte {
 
 	return bytes.Join(caBundles, []byte("\n"))
 }
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) VerifyOptions() x509.VerifyOptions {
+	// TODO make more efficient.  This isn't actually used in any of our mainline paths.  It's called to build the TLSConfig
+	// TODO on file changes, but the actual authentication runs against the individual items, not the union.
+	ret, err := newCABundleAndVerifier(c.Name(), c.CurrentCABundleContent())
+	if err != nil {
+		// because we're made up of already vetted values, this indicates some kind of coding error
+		panic(err)
+	}
+
+	return ret.verifyOptions
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) AddListener(listener Listener) {
+	for _, curr := range c {
+		if notifier, ok := curr.(Notifier); ok {
+			notifier.AddListener(listener)
+		}
+	}
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) RunOnce() error {
+	errors := []error{}
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			if err := controller.RunOnce(); err != nil {
+				errors = append(errors, err)
+			}
+		}
+	}
+
+	return utilerrors.NewAggregate(errors)
+}
+
+// Run runs the controller
+func (c unionCAContent) Run(workers int, stopCh <-chan struct{}) {
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			go controller.Run(workers, stopCh)
+		}
+	}
+}

pkg/server/options/authentication.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+
 	"github.com/spf13/pflag"
 
 	v1 "k8s.io/api/core/v1"
@@ -30,12 +32,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
-	"k8s.io/apiserver/pkg/authentication/request/x509"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/util/cert"
 	"k8s.io/klog"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 )
@@ -112,7 +112,7 @@ func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig
 		return nil, nil
 	}
 
-	verifyFn, err := x509.NewStaticVerifierFromFile(s.ClientCAFile)
+	caBundleProvider, err := dynamiccertificates.NewDynamicCAContentFromFile("request-header", s.ClientCAFile)
 	if err != nil {
 		return nil, err
 	}
@@ -121,7 +121,7 @@ func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig
 		UsernameHeaders:     headerrequest.StaticStringSlice(s.UsernameHeaders),
 		GroupHeaders:        headerrequest.StaticStringSlice(s.GroupHeaders),
 		ExtraHeaderPrefixes: headerrequest.StaticStringSlice(s.ExtraHeaderPrefixes),
-		VerifyOptionFn:      verifyFn,
+		CAContentProvider:   caBundleProvider,
 		AllowedClientNames:  headerrequest.StaticStringSlice(s.AllowedNames),
 	}, nil
 }
@@ -132,23 +132,23 @@ type ClientCertAuthenticationOptions struct {
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
 	ClientCA string
 
-	// ClientVerifyOptionFn are the options for verifying incoming connections using mTLS and directly assigning to users.
+	// CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
 	// Generally this is the CA bundle file used to authenticate client certificates
 	// If non-nil, this takes priority over the ClientCA file.
-	ClientVerifyOptionFn x509.VerifyOptionFunc
+	CAContentProvider dynamiccertificates.CAContentProvider
 }
 
 // GetClientVerifyOptionFn provides verify options for your authenticator while respecting the preferred order of verifiers.
-func (s *ClientCertAuthenticationOptions) GetClientVerifyOptionFn() (x509.VerifyOptionFunc, error) {
-	if s.ClientVerifyOptionFn != nil {
-		return s.ClientVerifyOptionFn, nil
+func (s *ClientCertAuthenticationOptions) GetClientCAContentProvider() (dynamiccertificates.CAContentProvider, error) {
+	if s.CAContentProvider != nil {
+		return s.CAContentProvider, nil
 	}
 
 	if len(s.ClientCA) == 0 {
 		return nil, nil
 	}
 
-	return x509.NewStaticVerifierFromFile(s.ClientCA)
+	return dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", s.ClientCA)
 }
 
 func (s *ClientCertAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
@@ -230,9 +230,9 @@ func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 		"Note that this can result in authentication that treats all requests as anonymous.")
 }
 
-func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
+func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
 	if s == nil {
-		c.Authenticator = nil
+		authenticationInfo.Authenticator = nil
 		return nil
 	}
 
@@ -266,20 +266,24 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo,
 	}
 
 	// configure AuthenticationInfo config
-	cfg.ClientVerifyOptionFn, err = s.ClientCert.GetClientVerifyOptionFn()
+	cfg.ClientCertificateCAContentProvider, err = s.ClientCert.GetClientCAContentProvider()
 	if err != nil {
 		return fmt.Errorf("unable to load client CA file: %v", err)
 	}
-	if err = c.ApplyClientCert(s.ClientCert.ClientCA, servingInfo); err != nil {
-		return fmt.Errorf("unable to load client CA file: %v", err)
+	if cfg.ClientCertificateCAContentProvider != nil {
+		if err = authenticationInfo.ApplyClientCert(cfg.ClientCertificateCAContentProvider, servingInfo); err != nil {
+			return fmt.Errorf("unable to load client CA file: %v", err)
+		}
 	}
 
 	cfg.RequestHeaderConfig, err = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
 	if err != nil {
 		return fmt.Errorf("unable to create request header authentication config: %v", err)
 	}
-	if err = c.ApplyClientCert(s.RequestHeader.ClientCAFile, servingInfo); err != nil {
-		return fmt.Errorf("unable to load client CA file: %v", err)
+	if cfg.RequestHeaderConfig != nil {
+		if err = authenticationInfo.ApplyClientCert(cfg.RequestHeaderConfig.CAContentProvider, servingInfo); err != nil {
+			return fmt.Errorf("unable to load client CA file: %v", err)
+		}
 	}
 
 	// create authenticator
@@ -287,11 +291,11 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo,
 	if err != nil {
 		return err
 	}
-	c.Authenticator = authenticator
+	authenticationInfo.Authenticator = authenticator
 	if openAPIConfig != nil {
 		openAPIConfig.SecurityDefinitions = securityDefinitions
 	}
-	c.SupportsBasicAuth = false
+	authenticationInfo.SupportsBasicAuth = false
 
 	return nil
 }
@@ -372,28 +376,14 @@ func inClusterClientCA(authConfigMap *v1.ConfigMap) (*ClientCertAuthenticationOp
 		// not having a client-ca is fine, return nil
 		return nil, nil
 	}
-
-	clientCAs, err := cert.NewPoolFromBytes([]byte(clientCA))
+	clientCAProvider, err := dynamiccertificates.NewStaticCAContent("client-ca-file", []byte(clientCA))
 	if err != nil {
-		return nil, fmt.Errorf("unable to load client CA from configmap: %v", err)
-	}
-	verifyOpts := x509.DefaultVerifyOptions()
-	verifyOpts.Roots = clientCAs
-
-	// we still need to write out the client-ca-file for now because it is used to plumb the options through the apiserver's
-	// configuration to hint clients.
-	// TODO deads2k this should eventually be made dynamic along with the authenticator.  I'm just wiring them one at at time.
-	f, err := ioutil.TempFile("", "client-ca-file")
-	if err != nil {
-		return nil, err
-	}
-	if err := ioutil.WriteFile(f.Name(), []byte(clientCA), 0600); err != nil {
 		return nil, err
 	}
 
 	return &ClientCertAuthenticationOptions{
-		ClientCA:             f.Name(),
-		ClientVerifyOptionFn: x509.StaticVerifierFn(verifyOpts),
+		ClientCA:          "",
+		CAContentProvider: clientCAProvider,
 	}, nil
 }
 

pkg/server/options/authentication_test.go

@@ -57,7 +57,7 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
 				UsernameHeaders:     headerrequest.StaticStringSlice{"x-remote-user"},
 				GroupHeaders:        headerrequest.StaticStringSlice{"x-remote-group"},
 				ExtraHeaderPrefixes: headerrequest.StaticStringSlice{"x-remote-extra-"},
-				VerifyOptionFn:      nil, // this is nil because you can't compare functions
+				CAContentProvider:   nil, // this is nil because you can't compare functions
 				AllowedClientNames:  headerrequest.StaticStringSlice{"kube-aggregator"},
 			},
 		},
@@ -70,10 +70,10 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
 				t.Fatal(err)
 			}
 			if resultConfig != nil {
-				if resultConfig.VerifyOptionFn == nil {
+				if resultConfig.CAContentProvider == nil {
 					t.Error("missing requestheader verify")
 				}
-				resultConfig.VerifyOptionFn = nil
+				resultConfig.CAContentProvider = nil
 			}
 
 			if !reflect.DeepEqual(resultConfig, testcase.expectConfig) {

pkg/server/secure_serving.go

@@ -72,6 +72,20 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
 			s.SNICerts,
 			nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages.
 		)
+		// register if possible
+		if notifier, ok := s.ClientCA.(dynamiccertificates.Notifier); ok {
+			notifier.AddListener(dynamicCertificateController)
+		}
+		// start controllers if possible
+		if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok {
+			// runonce to be sure that we have a value.
+			if err := controller.RunOnce(); err != nil {
+				return nil, err
+			}
+
+			go controller.Run(1, stopCh)
+		}
+
 		// runonce to be sure that we have a value.
 		if err := dynamicCertificateController.RunOnce(); err != nil {
 			return nil, err