credentials: Add certificate provider plugin APIs. (#3654)

Also add an implementation for the `Distributor` type which makes it easier to implement new `Provider` types.

credentials/tls/certprovider/distributor.go

@@ -0,0 +1,110 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"context"
+	"sync"
+
+	"google.golang.org/grpc/internal/grpcsync"
+)
+
+// Distributor makes it easy for provider implementations to furnish new key
+// materials by handling synchronization between the producer and consumers of
+// the key material.
+//
+// Provider implementations which choose to use a Distributor should do the
+// following:
+// - create a new Distributor using the NewDistributor() function.
+// - invoke the Set() method whenever they have new key material or errors to
+//   report.
+// - delegate to the distributor when handing calls to KeyMaterial().
+// - invoke the Stop() method when they are done using the distributor.
+type Distributor struct {
+	// mu protects the underlying key material.
+	mu   sync.Mutex
+	km   *KeyMaterial
+	pErr error
+
+	ready  *grpcsync.Event
+	closed *grpcsync.Event
+}
+
+// NewDistributor returns a new Distributor.
+func NewDistributor() *Distributor {
+	return &Distributor{
+		ready:  grpcsync.NewEvent(),
+		closed: grpcsync.NewEvent(),
+	}
+}
+
+// Set updates the key material in the distributor with km.
+//
+// Provider implementations which use the distributor must not modify the
+// contents of the KeyMaterial struct pointed to by km.
+//
+// A non-nil err value indicates the error that the provider implementation ran
+// into when trying to fetch key material, and makes it possible to surface the
+// error to the user. A non-nil error value passed here causes distributor's
+// KeyMaterial() method to return nil key material.
+func (d *Distributor) Set(km *KeyMaterial, err error) {
+	d.mu.Lock()
+	d.km = km
+	d.pErr = err
+	if err != nil {
+		// If a non-nil err is passed, we ignore the key material being passed.
+		d.km = nil
+	}
+	d.ready.Fire()
+	d.mu.Unlock()
+}
+
+// KeyMaterial returns the most recent key material provided to the distributor.
+// If no key material was provided at the time of this call, it will block until
+// the deadline on the context expires or fresh key material arrives.
+func (d *Distributor) KeyMaterial(ctx context.Context) (*KeyMaterial, error) {
+	if d.closed.HasFired() {
+		return nil, errProviderClosed
+	}
+
+	if d.ready.HasFired() {
+		return d.keyMaterial()
+	}
+
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-d.closed.Done():
+		return nil, errProviderClosed
+	case <-d.ready.Done():
+		return d.keyMaterial()
+	}
+}
+
+func (d *Distributor) keyMaterial() (*KeyMaterial, error) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.km, d.pErr
+}
+
+// Stop turns down the distributor, releases allocated resources and fails any
+// active KeyMaterial() call waiting for new key material.
+func (d *Distributor) Stop() {
+	d.closed.Fire()
+}

credentials/tls/certprovider/distributor_test.go

@@ -0,0 +1,159 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+var errProviderTestInternal = errors.New("provider internal error")
+
+// TestDistributor invokes the different methods on the Distributor type and
+// verifies the results.
+func (s) TestDistributor(t *testing.T) {
+	dist := NewDistributor()
+
+	// Read cert/key files from testdata.
+	km, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// wantKM1 has both local and root certs.
+	wantKM1 := *km
+	// wantKM2 has only local certs. Roots are nil-ed out.
+	wantKM2 := *km
+	wantKM2.Roots = nil
+
+	// Create a goroutines which work in lockstep with the rest of the test.
+	// This goroutine reads the key material from the distributor while the rest
+	// of the test sets it.
+	var wg sync.WaitGroup
+	wg.Add(1)
+	errCh := make(chan error)
+	proceedCh := make(chan struct{})
+	go func() {
+		defer wg.Done()
+
+		// The first call to KeyMaterial() should timeout because no key
+		// material has been set on the distributor as yet.
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout/2)
+		defer cancel()
+		if _, err := dist.KeyMaterial(ctx); err != context.DeadlineExceeded {
+			errCh <- err
+			return
+		}
+		proceedCh <- struct{}{}
+
+		// This call to KeyMaterial() should return the key material with both
+		// the local certs and the root certs.
+		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		gotKM, err := dist.KeyMaterial(ctx)
+		if err != nil {
+			errCh <- err
+			return
+		}
+		if !reflect.DeepEqual(gotKM, &wantKM1) {
+			errCh <- fmt.Errorf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM1)
+		}
+		proceedCh <- struct{}{}
+
+		// This call to KeyMaterial() should eventually return key material with
+		// only the local certs.
+		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		for {
+			gotKM, err := dist.KeyMaterial(ctx)
+			if err != nil {
+				errCh <- err
+				return
+			}
+			if reflect.DeepEqual(gotKM, &wantKM2) {
+				break
+			}
+		}
+		proceedCh <- struct{}{}
+
+		// This call to KeyMaterial() should return nil key material and a
+		// non-nil error.
+		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		for {
+			gotKM, err := dist.KeyMaterial(ctx)
+			if gotKM == nil && err == errProviderTestInternal {
+				break
+			}
+			if err != nil {
+				// If we have gotten any error other than
+				// errProviderTestInternal, we should bail out.
+				errCh <- err
+				return
+			}
+		}
+		proceedCh <- struct{}{}
+
+		// This call to KeyMaterial() should eventually return errProviderClosed
+		// error.
+		ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		for {
+			if _, err := dist.KeyMaterial(ctx); err == errProviderClosed {
+				break
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+	}()
+
+	waitAndDo(t, proceedCh, errCh, func() {
+		dist.Set(&wantKM1, nil)
+	})
+
+	waitAndDo(t, proceedCh, errCh, func() {
+		dist.Set(&wantKM2, nil)
+	})
+
+	waitAndDo(t, proceedCh, errCh, func() {
+		dist.Set(&wantKM2, errProviderTestInternal)
+	})
+
+	waitAndDo(t, proceedCh, errCh, func() {
+		dist.Stop()
+	})
+
+}
+
+func waitAndDo(t *testing.T, proceedCh chan struct{}, errCh chan error, do func()) {
+	t.Helper()
+
+	timer := time.NewTimer(defaultTestTimeout)
+	select {
+	case <-timer.C:
+		t.Fatalf("test timed out when waiting for event from distributor")
+	case <-proceedCh:
+		do()
+	case err := <-errCh:
+		t.Fatal(err)
+	}
+}

credentials/tls/certprovider/provider.go

@@ -0,0 +1,104 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package certprovider defines APIs for certificate providers in gRPC.
+//
+// Experimental
+//
+// Notice: All APIs in this package are experimental and may be removed in a
+// later release.
+package certprovider
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+)
+
+var (
+	// errProviderClosed is returned by Distributor.KeyMaterial when it is
+	// closed.
+	errProviderClosed = errors.New("provider instance is closed")
+
+	// m is a map from name to provider builder.
+	m = make(map[string]Builder)
+)
+
+// Register registers the provider builder, whose name as returned by its Name()
+// method will be used as the name registered with this builder. Registered
+// Builders are used by the Store to create Providers.
+func Register(b Builder) {
+	m[b.Name()] = b
+}
+
+// getBuilder returns the provider builder registered with the given name.
+// If no builder is registered with the provided name, nil will be returned.
+func getBuilder(name string) Builder {
+	if b, ok := m[name]; ok {
+		return b
+	}
+	return nil
+}
+
+// Builder creates a Provider.
+type Builder interface {
+	// Build creates a new provider with the provided config.
+	Build(StableConfig) Provider
+
+	// ParseConfig converts config input in a format specific to individual
+	// implementations and returns an implementation of the StableConfig
+	// interface.
+	// Equivalent configurations must return StableConfig types whose
+	// Canonical() method returns the same output.
+	ParseConfig(interface{}) (StableConfig, error)
+
+	// Name returns the name of providers built by this builder.
+	Name() string
+}
+
+// StableConfig wraps the method to return a stable provider configuration.
+type StableConfig interface {
+	// Canonical returns provider config as an arbitrary byte slice.
+	// Equivalent configurations must return the same output.
+	Canonical() []byte
+}
+
+// Provider makes it possible to keep channel credential implementations up to
+// date with secrets that they rely on to secure communications on the
+// underlying channel.
+//
+// Provider implementations are free to rely on local or remote sources to fetch
+// the latest secrets, and free to share any state between different
+// instantiations as they deem fit.
+type Provider interface {
+	// KeyMaterial returns the key material sourced by the provider.
+	// Callers are expected to use the returned value as read-only.
+	KeyMaterial(ctx context.Context) (*KeyMaterial, error)
+
+	// Close cleans up resources allocated by the provider.
+	Close()
+}
+
+// KeyMaterial wraps the certificates and keys returned by a provider instance.
+type KeyMaterial struct {
+	// Certs contains a slice of cert/key pairs used to prove local identity.
+	Certs []tls.Certificate
+	// Roots contains the set of trusted roots to validate the peer's identity.
+	Roots *x509.CertPool
+}

credentials/tls/certprovider/store.go

@@ -0,0 +1,120 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"fmt"
+	"sync"
+)
+
+// provStore is the global singleton certificate provider store.
+var provStore = &store{
+	providers: make(map[storeKey]*wrappedProvider),
+}
+
+// storeKey acts as the key to the map of providers maintained by the store. A
+// combination of provider name and configuration is used to uniquely identify
+// every provider instance in the store. Go maps need to be indexed by
+// comparable types, so the provider configuration is converted from
+// `interface{}` to string using the ParseConfig method while creating this key.
+type storeKey struct {
+	// name of the certificate provider.
+	name string
+	// configuration of the certificate provider in string form.
+	config string
+}
+
+// wrappedProvider wraps a provider instance with a reference count.
+type wrappedProvider struct {
+	Provider
+	refCount int
+
+	// A reference to the key and store are also kept here to override the
+	// Close method on the provider.
+	storeKey storeKey
+	store    *store
+}
+
+// store is a collection of provider instances, safe for concurrent access.
+type store struct {
+	mu        sync.Mutex
+	providers map[storeKey]*wrappedProvider
+}
+
+// GetProvider returns a provider instance corresponding to name and config.
+// name is the registered name of the provider and config is the
+// provider-specific configuration. Implementations of the Builder interface
+// should clearly document the type of configuration accepted by them.
+//
+// If a provider exists for the (name+config) combination, its reference count
+// is incremented before returning. If no provider exists for the (name+config)
+// combination, a new one is created using the registered builder. If no
+// registered builder is found, or the provider configuration is rejected by it,
+// a non-nil error is returned.
+func GetProvider(name string, config interface{}) (Provider, error) {
+	provStore.mu.Lock()
+	defer provStore.mu.Unlock()
+
+	builder := getBuilder(name)
+	if builder == nil {
+		return nil, fmt.Errorf("no registered builder for provider name: %s", name)
+	}
+	stableConfig, err := builder.ParseConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
+	sk := storeKey{
+		name:   name,
+		config: string(stableConfig.Canonical()),
+	}
+	if wp, ok := provStore.providers[sk]; ok {
+		wp.refCount++
+		return wp, nil
+	}
+
+	provider := builder.Build(stableConfig)
+	if provider == nil {
+		return nil, fmt.Errorf("certprovider.Build(%v) failed", sk)
+	}
+	wp := &wrappedProvider{
+		Provider: provider,
+		refCount: 1,
+		storeKey: sk,
+		store:    provStore,
+	}
+	provStore.providers[sk] = wp
+	return wp, nil
+}
+
+// Close overrides the Close method of the embedded provider. It releases the
+// reference held by the caller on the underlying provider and if the
+// provider's reference count reaches zero, it is removed from the store, and
+// its Close method is also invoked.
+func (wp *wrappedProvider) Close() {
+	ps := wp.store
+	ps.mu.Lock()
+	defer ps.mu.Unlock()
+
+	wp.refCount--
+	if wp.refCount == 0 {
+		wp.Provider.Close()
+		delete(ps.providers, wp.storeKey)
+	}
+}

credentials/tls/certprovider/store_test.go

@@ -0,0 +1,351 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package certprovider
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"reflect"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/testdata"
+)
+
+const (
+	fakeProvider1Name  = "fake-certificate-provider-1"
+	fakeProvider2Name  = "fake-certificate-provider-2"
+	fakeConfig         = "my fake config"
+	defaultTestTimeout = 1 * time.Second
+)
+
+func init() {
+	Register(&fakeProviderBuilder{name: fakeProvider1Name})
+	Register(&fakeProviderBuilder{name: fakeProvider2Name})
+}
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// fakeProviderBuilder builds new instances of fakeProvider and interprets the
+// config provided to it as a string.
+type fakeProviderBuilder struct {
+	name string
+}
+
+func (b *fakeProviderBuilder) Build(StableConfig) Provider {
+	ctx, cancel := context.WithCancel(context.Background())
+	p := &fakeProvider{
+		Distributor: NewDistributor(),
+		cancel:      cancel,
+		done:        make(chan struct{}),
+		kmCh:        make(chan *KeyMaterial, 2),
+	}
+	go p.run(ctx)
+	return p
+}
+
+func (b *fakeProviderBuilder) ParseConfig(config interface{}) (StableConfig, error) {
+	s, ok := config.(string)
+	if !ok {
+		return nil, fmt.Errorf("provider %s received bad config %v", b.name, config)
+	}
+	return &fakeStableConfig{config: s}, nil
+}
+
+func (b *fakeProviderBuilder) Name() string {
+	return b.name
+}
+
+type fakeStableConfig struct {
+	config string
+}
+
+func (c *fakeStableConfig) Canonical() []byte {
+	return []byte(c.config)
+}
+
+// fakeProvider is an implementation of the Provider interface which embeds a
+// Distributor and exposes two channels for the user:
+// 1. to be notified when the provider is closed
+// 2. to push new key material into the provider
+type fakeProvider struct {
+	*Distributor
+
+	// Used to cancel the run goroutine when the provider is closed.
+	cancel context.CancelFunc
+	// This channel is closed when the provider is closed. Tests should block on
+	// this to make sure the provider is closed.
+	done chan struct{}
+	// Tests can push new key material on this channel, and the provider will
+	// return this on subsequent calls to KeyMaterial().
+	kmCh chan *KeyMaterial
+}
+
+func (p *fakeProvider) run(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case km := <-p.kmCh:
+			p.Distributor.Set(km, nil)
+		}
+	}
+}
+
+func (p *fakeProvider) Close() {
+	p.cancel()
+	p.Distributor.Stop()
+}
+
+// loadKeyMaterials is a helper to read cert/key files from testdata and convert
+// them into a KeyMaterial struct.
+func loadKeyMaterials() (*KeyMaterial, error) {
+	certs, err := tls.LoadX509KeyPair(testdata.Path("server1.pem"), testdata.Path("server1.key"))
+	if err != nil {
+		return nil, err
+	}
+
+	pemData, err := ioutil.ReadFile(testdata.Path("ca.pem"))
+	if err != nil {
+		return nil, err
+	}
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM(pemData)
+	return &KeyMaterial{Certs: []tls.Certificate{certs}, Roots: roots}, nil
+}
+
+func makeProvider(t *testing.T, name, config string) (Provider, *fakeProvider) {
+	t.Helper()
+
+	prov, err := GetProvider(name, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// The store returns a wrappedProvider, which holds a reference to the
+	// actual provider, which in our case in the fakeProvider.
+	wp := prov.(*wrappedProvider)
+	fp := wp.Provider.(*fakeProvider)
+	return prov, fp
+}
+
+// TestStoreWithSingleProvider creates a single provider through the store and
+// calls methods on it.
+func (s) TestStoreWithSingleProvider(t *testing.T) {
+	prov, fp := makeProvider(t, fakeProvider1Name, fakeConfig)
+
+	// Push key materials into the provider.
+	wantKM, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	fp.kmCh <- wantKM
+
+	// Get key materials from the provider and compare it to the ones we pushed
+	// above.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotKM, err := prov.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	// TODO(easwars): Remove all references to reflect.DeepEqual and use
+	// cmp.Equal instead. Currently, the later panics because x509.Certificate
+	// type defines an Equal method, but does not check for nil. This has been
+	// fixed in
+	// https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351,
+	// but this is only available starting go1.14. So, once we remove support
+	// for go1.13, we can make the switch.
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Close the provider and retry the KeyMaterial() call, and expect it to
+	// fail with a known error.
+	prov.Close()
+	if _, err := prov.KeyMaterial(ctx); err != errProviderClosed {
+		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	}
+}
+
+// TestStoreWithSingleProviderWithSharing creates multiple instances of the same
+// type of provider through the store (and expects the store's sharing mechanism
+// to kick in) and calls methods on it.
+func (s) TestStoreWithSingleProviderWithSharing(t *testing.T) {
+	prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig)
+	prov2, _ := makeProvider(t, fakeProvider1Name, fakeConfig)
+
+	// Push key materials into the fake provider1.
+	wantKM, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	fp1.kmCh <- wantKM
+
+	// Get key materials from the fake provider2 and compare it to the ones we
+	// pushed above.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotKM, err := prov2.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Close the provider1 and retry the KeyMaterial() call on prov2, and expect
+	// it to succeed.
+	prov1.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+
+	prov2.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed {
+		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	}
+}
+
+// TestStoreWithSingleProviderWithoutSharing creates multiple instances of the
+// same type of provider through the store with different configs. The store
+// would end up creating different provider instances for these and no sharing
+// would take place.
+func (s) TestStoreWithSingleProviderWithoutSharing(t *testing.T) {
+	prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig+"1111")
+	prov2, fp2 := makeProvider(t, fakeProvider1Name, fakeConfig+"2222")
+
+	// Push the same key materials into the two providers.
+	wantKM, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	fp1.kmCh <- wantKM
+	fp2.kmCh <- wantKM
+
+	// Get key materials from the fake provider1 and compare it to the ones we
+	// pushed above.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotKM, err := prov1.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Get key materials from the fake provider2 and compare it to the ones we
+	// pushed above.
+	gotKM, err = prov2.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Update the key materials used by provider1, and make sure provider2 is
+	// not affected.
+	newKM, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	newKM.Roots = nil
+	fp1.kmCh <- newKM
+
+	gotKM, err = prov2.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Close the provider1 and retry the KeyMaterial() call on prov2, and expect
+	// it to succeed.
+	prov1.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+
+	prov2.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed {
+		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	}
+}
+
+// TestStoreWithMultipleProviders creates multiple providers of different types
+// and make sure closing of one does not affect the other.
+func (s) TestStoreWithMultipleProviders(t *testing.T) {
+	prov1, fp1 := makeProvider(t, fakeProvider1Name, fakeConfig)
+	prov2, fp2 := makeProvider(t, fakeProvider2Name, fakeConfig)
+
+	// Push key materials into the fake providers.
+	wantKM, err := loadKeyMaterials()
+	if err != nil {
+		t.Fatal(err)
+	}
+	fp1.kmCh <- wantKM
+	fp2.kmCh <- wantKM
+
+	// Get key materials from the fake provider1 and compare it to the ones we
+	// pushed above.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	gotKM, err := prov1.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Get key materials from the fake provider2 and compare it to the ones we
+	// pushed above.
+	gotKM, err = prov2.KeyMaterial(ctx)
+	if err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+	if !reflect.DeepEqual(gotKM, wantKM) {
+		t.Fatalf("provider.KeyMaterial() = %+v, want %+v", gotKM, wantKM)
+	}
+
+	// Close the provider1 and retry the KeyMaterial() call on prov2, and expect
+	// it to succeed.
+	prov1.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != nil {
+		t.Fatalf("provider.KeyMaterial() = %v", err)
+	}
+
+	prov2.Close()
+	if _, err := prov2.KeyMaterial(ctx); err != errProviderClosed {
+		t.Fatalf("provider.KeyMaterial() = %v, wantErr: %v", err, errProviderClosed)
+	}
+}