grpc-go/internal/xds/bootstrap/tlscreds/bundle_ext_test.go

521 lines
18 KiB
Go

/*
*
* Copyright 2023 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package tlscreds_test
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"os"
"strings"
"testing"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/internal/envconfig"
"google.golang.org/grpc/internal/grpctest"
"google.golang.org/grpc/internal/stubserver"
"google.golang.org/grpc/internal/testutils"
"google.golang.org/grpc/internal/xds/bootstrap/tlscreds"
testgrpc "google.golang.org/grpc/interop/grpc_testing"
testpb "google.golang.org/grpc/interop/grpc_testing"
"google.golang.org/grpc/status"
"google.golang.org/grpc/testdata"
)
const defaultTestTimeout = 5 * time.Second
type s struct {
grpctest.Tester
}
func Test(t *testing.T) {
grpctest.RunSubTests(t, s{})
}
type Closable interface {
Close()
}
func (s) TestValidTlsBuilder(t *testing.T) {
caCert := testdata.Path("x509/server_ca_cert.pem")
clientCert := testdata.Path("x509/client1_cert.pem")
clientKey := testdata.Path("x509/client1_key.pem")
clientSpiffeBundle := testdata.Path("spiffe_end2end/client_spiffebundle.json")
tests := []struct {
name string
jd string
}{
{
name: "Absent configuration",
jd: `null`,
},
{
name: "Empty configuration",
jd: `{}`,
},
{
name: "Only CA certificate chain",
jd: fmt.Sprintf(`{"ca_certificate_file": "%s"}`, caCert),
},
{
name: "Only private key and certificate chain",
jd: fmt.Sprintf(`{"certificate_file":"%s","private_key_file":"%s"}`, clientCert, clientKey),
},
{
name: "CA chain, private key and certificate chain",
jd: fmt.Sprintf(`{"ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s"}`, caCert, clientCert, clientKey),
},
{
name: "Only refresh interval", jd: `{"refresh_interval": "1s"}`,
},
{
name: "Refresh interval and CA certificate chain",
jd: fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file": "%s"}`, caCert),
},
{
name: "Refresh interval, private key and certificate chain",
jd: fmt.Sprintf(`{"refresh_interval": "1s","certificate_file":"%s","private_key_file":"%s"}`, clientCert, clientKey),
},
{
name: "Refresh interval, CA chain, private key and certificate chain",
jd: fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s"}`, caCert, clientCert, clientKey),
},
{
name: "Refresh interval, CA chain, private key, certificate chain, spiffe bundle",
jd: fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s","spiffe_trust_bundle_map_file":"%s"}`, caCert, clientCert, clientKey, clientSpiffeBundle),
},
{
name: "Unknown field",
jd: `{"unknown_field": "foo"}`,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
msg := json.RawMessage(test.jd)
_, stop, err := tlscreds.NewBundle(msg)
if err != nil {
t.Fatalf("NewBundle(%s) returned error %s when expected to succeed", test.jd, err)
}
stop()
})
}
}
func (s) TestInvalidTlsBuilder(t *testing.T) {
tests := []struct {
name, jd, wantErrPrefix string
}{
{
name: "Wrong type in json",
jd: `{"ca_certificate_file": 1}`,
wantErrPrefix: "failed to unmarshal config:"},
{
name: "Missing private key",
jd: fmt.Sprintf(`{"certificate_file":"%s"}`, testdata.Path("x509/server_cert.pem")),
wantErrPrefix: "pemfile: private key file and identity cert file should be both specified or not specified",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
msg := json.RawMessage(test.jd)
_, stop, err := tlscreds.NewBundle(msg)
if err == nil || !strings.HasPrefix(err.Error(), test.wantErrPrefix) {
if stop != nil {
stop()
}
t.Fatalf("NewBundle(%s): got error %s, want an error with prefix %s", msg, err, test.wantErrPrefix)
}
})
}
}
func (s) TestCaReloading(t *testing.T) {
serverCa, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
if err != nil {
t.Fatalf("Failed to read test CA cert: %s", err)
}
// Write CA certs to a temporary file so that we can modify it later.
caPath := t.TempDir() + "/ca.pem"
if err = os.WriteFile(caPath, serverCa, 0644); err != nil {
t.Fatalf("Failed to write test CA cert: %v", err)
}
cfg := fmt.Sprintf(`{
"ca_certificate_file": "%s",
"refresh_interval": ".01s"
}`, caPath)
tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
if err != nil {
t.Fatalf("Failed to create TLS bundle: %v", err)
}
defer stop()
serverCredentials := grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.NoClientCert))
server := stubserver.StartTestService(t, nil, serverCredentials)
conn, err := grpc.NewClient(
server.Address,
grpc.WithCredentialsBundle(tlsBundle),
grpc.WithAuthority("x.test.example.com"),
)
if err != nil {
t.Fatalf("Error dialing: %v", err)
}
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
client := testgrpc.NewTestServiceClient(conn)
if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
t.Errorf("Error calling EmptyCall: %v", err)
}
// close the server and create a new one to force client to do a new
// handshake.
server.Stop()
invalidCa, err := os.ReadFile(testdata.Path("ca.pem"))
if err != nil {
t.Fatalf("Failed to read test CA cert: %v", err)
}
// unload root cert
err = os.WriteFile(caPath, invalidCa, 0644)
if err != nil {
t.Fatalf("Failed to write test CA cert: %v", err)
}
for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) {
ss := stubserver.StubServer{
Address: server.Address,
EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
}
server = stubserver.StartTestService(t, &ss, serverCredentials)
// Client handshake should eventually fail because the client CA was
// reloaded, and thus the server cert is signed by an unknown CA.
t.Log(server)
_, err = client.EmptyCall(ctx, &testpb.Empty{})
const wantErr = "certificate signed by unknown authority"
if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) {
// Certs have reloaded.
server.Stop()
break
}
t.Logf("EmptyCall() got err: %s, want code: %s, want err: %s", err, codes.Unavailable, wantErr)
server.Stop()
}
if ctx.Err() != nil {
t.Errorf("Timed out waiting for CA certs reloading")
}
}
// Test_SPIFFE_Reloading sets up a client and server. The client is configured
// to use a SPIFFE bundle map, and the server is configured to use TLS creds
// compatible with this bundle. A handshake is performed and connection is
// expected to be successful. Then we change the client's SPIFFE Bundle Map file
// on disk to one that should fail with the server's credentials. This change
// should be picked up by the client via our file reloading. Another handshake
// is performed and checked for failure, ensuring that gRPC is correctly using
// the changed-on-disk bundle map.
func (s) Test_SPIFFE_Reloading(t *testing.T) {
testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true)
clientSPIFFEBundle, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffebundle.json"))
if err != nil {
t.Fatalf("Failed to read test SPIFFE bundle: %v", err)
}
// Write CA certs to a temporary file so that we can modify it later.
spiffePath := t.TempDir() + "/client_spiffe.json"
if err = os.WriteFile(spiffePath, clientSPIFFEBundle, 0644); err != nil {
t.Fatalf("Failed to write test SPIFFE Bundle %v: %v", clientSPIFFEBundle, err)
}
cfg := fmt.Sprintf(`{
"spiffe_trust_bundle_map_file": "%s",
"refresh_interval": ".01s"
}`, spiffePath)
tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
if err != nil {
t.Fatalf("Failed to create TLS bundle: %v", err)
}
defer stop()
l, err := testutils.LocalTCPListener()
if err != nil {
t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
}
lis := testutils.NewRestartableListener(l)
defer lis.Close()
ss := stubserver.StubServer{
Listener: lis,
EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
}
serverCredentials := grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.NoClientCert))
server := stubserver.StartTestService(t, &ss, serverCredentials)
defer server.Stop()
conn, err := grpc.NewClient(
server.Address,
grpc.WithCredentialsBundle(tlsBundle),
grpc.WithAuthority("x.test.example.com"),
)
if err != nil {
t.Fatalf("grpc.NewClient(%q) failed: %v", server.Address, err)
}
defer conn.Close()
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
client := testgrpc.NewTestServiceClient(conn)
if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
t.Errorf("Error calling EmptyCall: %v", err)
}
// Setup the wrong bundle to be reloaded
wrongBundle, err := os.ReadFile(testdata.Path("spiffe_end2end/server_spiffebundle.json"))
if err != nil {
t.Fatalf("Failed to read test spiffe bundle %v: %v", "spiffe_end2end/server_spiffebundle.json", err)
}
// Write the bundle that will fail to the tmp file path to be reloaded
err = os.WriteFile(spiffePath, wrongBundle, 0644)
if err != nil {
t.Fatalf("Failed to write test spiffe bundle %v: %v", "spiffe_end2end/server_spiffebundle.json", err)
}
for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) {
// Stop and restart the listener to force new handshakes
lis.Stop()
lis.Restart()
// Client handshake should eventually fail because the client CA was
// reloaded, and thus the server cert is signed by an unknown CA.
t.Log(server)
_, err = client.EmptyCall(ctx, &testpb.Empty{})
const wantErr = "no bundle found for peer certificates trust domain"
if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) {
// Certs have reloaded.
server.Stop()
break
}
t.Logf("EmptyCall() got err: %s, want code: %s, want err: %s", err, codes.Unavailable, wantErr)
}
if ctx.Err() != nil {
t.Errorf("Timed out waiting for CA certs reloading")
}
}
func (s) TestMTLS(t *testing.T) {
s := stubserver.StartTestService(t, nil, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)))
defer s.Stop()
cfg := fmt.Sprintf(`{
"ca_certificate_file": "%s",
"certificate_file": "%s",
"private_key_file": "%s"
}`,
testdata.Path("x509/server_ca_cert.pem"),
testdata.Path("x509/client1_cert.pem"),
testdata.Path("x509/client1_key.pem"))
tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
if err != nil {
t.Fatalf("Failed to create TLS bundle: %v", err)
}
defer stop()
conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
if err != nil {
t.Fatalf("Error dialing: %v", err)
}
defer conn.Close()
client := testgrpc.NewTestServiceClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
t.Errorf("EmptyCall(): got error %v when expected to succeed", err)
}
}
// Test_MTLS_SPIFFE configures a client and server. The server has a certificate
// chain that is compatible with the client's configured SPIFFE bundle map. An
// MTLS connection is attempted between the two and checked for success.
func (s) Test_MTLS_SPIFFE(t *testing.T) {
testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true)
tests := []struct {
name string
serverOption grpc.ServerOption
}{
{
name: "MTLS SPIFFE",
serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)),
},
{
name: "MTLS SPIFFE Chain",
serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFEChain(t, tls.RequireAndVerifyClientCert)),
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
s := stubserver.StartTestService(t, nil, tc.serverOption)
defer s.Stop()
cfg := fmt.Sprintf(`{
"certificate_file": "%s",
"private_key_file": "%s",
"spiffe_trust_bundle_map_file": "%s"
}`,
testdata.Path("spiffe_end2end/client_spiffe.pem"),
testdata.Path("spiffe_end2end/client.key"),
testdata.Path("spiffe_end2end/client_spiffebundle.json"))
tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
if err != nil {
t.Fatalf("Failed to create TLS bundle: %v", err)
}
defer stop()
conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
if err != nil {
t.Fatalf("Error dialing: %v", err)
}
defer conn.Close()
client := testgrpc.NewTestServiceClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
t.Errorf("EmptyCall(): got error %v when expected to succeed", err)
}
})
}
}
// Test_MTLS_SPIFFE_FlagDisabled configures a client and server. The server has
// a certificate chain that is compatible with the client's configured SPIFFE
// bundle map. However, the XDS flag that enabled SPIFFE usage is disabled. An
// MTLS connection is attempted between the two and checked for failure.
func (s) Test_MTLS_SPIFFE_FlagDisabled(t *testing.T) {
testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, false)
serverOption := grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert))
s := stubserver.StartTestService(t, nil, serverOption)
defer s.Stop()
cfg := fmt.Sprintf(`{
"certificate_file": "%s",
"private_key_file": "%s",
"spiffe_trust_bundle_map_file": "%s"
}`,
testdata.Path("spiffe_end2end/client_spiffe.pem"),
testdata.Path("spiffe_end2end/client.key"),
testdata.Path("spiffe_end2end/client_spiffebundle.json"))
tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
if err != nil {
t.Fatalf("Failed to create TLS bundle: %v", err)
}
defer stop()
conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
if err != nil {
t.Fatalf("Error dialing: %v", err)
}
defer conn.Close()
client := testgrpc.NewTestServiceClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil {
t.Errorf("EmptyCall(): got success want failure")
}
}
func (s) Test_MTLS_SPIFFE_Failure(t *testing.T) {
testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true)
tests := []struct {
name string
certFile string
keyFile string
spiffeBundleFile string
serverOption grpc.ServerOption
wantErrContains string
wantErrCode codes.Code
}{
{
name: "No matching trust domain in bundle",
certFile: "spiffe_end2end/client_spiffe.pem",
keyFile: "spiffe_end2end/client.key",
spiffeBundleFile: "spiffe_end2end/server_spiffebundle.json",
serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)),
wantErrContains: "spiffe: no bundle found for peer certificates",
wantErrCode: codes.Unavailable,
},
{
name: "Server cert has no valid SPIFFE URIs",
certFile: "spiffe_end2end/client_spiffe.pem",
keyFile: "spiffe_end2end/client.key",
spiffeBundleFile: "spiffe_end2end/client_spiffebundle.json",
serverOption: grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)),
wantErrContains: "spiffe: could not get spiffe ID from peer leaf cert",
wantErrCode: codes.Unavailable,
},
{
name: "Server cert has valid spiffe ID but doesn't chain to the root CA",
certFile: "spiffe_end2end/client_spiffe.pem",
keyFile: "spiffe_end2end/client.key",
spiffeBundleFile: "spiffe_end2end/client_spiffebundle.json",
serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsValidSPIFFEButWrongCA(t, tls.RequireAndVerifyClientCert)),
wantErrContains: "spiffe: x509 certificate Verify failed: x509: certificate signed by unknown authority",
wantErrCode: codes.Unavailable,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
s := stubserver.StartTestService(t, nil, tc.serverOption)
defer s.Stop()
cfg := fmt.Sprintf(`{
"certificate_file": "%s",
"private_key_file": "%s",
"spiffe_trust_bundle_map_file": "%s"
}`,
testdata.Path(tc.certFile),
testdata.Path(tc.keyFile),
testdata.Path(tc.spiffeBundleFile))
tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
if err != nil {
t.Fatalf("Failed to create TLS bundle: %v", err)
}
defer stop()
conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
if err != nil {
t.Fatalf("grpc.NewClient(%q) failed: %v", s.Address, err)
}
defer conn.Close()
client := testgrpc.NewTestServiceClient(conn)
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()
if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil {
t.Errorf("EmptyCall(): got success. want failure")
}
if status.Code(err) != tc.wantErrCode {
t.Errorf("EmptyCall(): failed with wrong error. got code %v. want code: %v", status.Code(err), tc.wantErrCode)
}
if !strings.Contains(err.Error(), tc.wantErrContains) {
t.Errorf("EmptyCall(): failed with wrong error. got %v. want contains: %v", err, tc.wantErrContains)
}
})
}
}