letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow forwarding of nonce-service Redeem RPCs from one service… (#4297) 

Fixes #4295.
This commit is contained in:
Roland Bracewell Shoemaker 2019-06-26 13:04:31 -07:00 committed by GitHub
parent 352899ba2f
commit 844ae26b65
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 230 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
cmd/nonce-service/main.go
View File

@ -4,10 +4,13 @@ import (
"context"
"errors"
"flag"
"sync"
"time"
"github.com/letsencrypt/boulder/cmd"
corepb "github.com/letsencrypt/boulder/core/proto"
bgrpc "github.com/letsencrypt/boulder/grpc"
blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/nonce"
noncepb "github.com/letsencrypt/boulder/nonce/proto"
)
@ -17,18 +20,69 @@ type config struct {
cmd.ServiceConfig
Syslog cmd.SyslogConfig
MaxUsed int
RemoteNonceServices []cmd.GRPCClientConfig
}
}
type nonceServer struct {
inner *nonce.NonceService
inner *nonce.NonceService
remoteServices []noncepb.NonceServiceClient
log blog.Logger
}
func (ns *nonceServer) Redeem(_ context.Context, msg *noncepb.NonceMessage) (*noncepb.ValidMessage, error) {
func (ns *nonceServer) remoteRedeem(ctx context.Context, msg *noncepb.NonceMessage) bool {
deadline, ok := ctx.Deadline()
if !ok {
ns.log.Err("Context passed to remoteRedeem does not have a deadline")
return false
}
subCtx, cancel := context.WithDeadline(ctx, deadline.Add(-time.Millisecond*250))
defer cancel()
forwarded := true
msg.Forwarded = &forwarded
results := make(chan bool, len(ns.remoteServices))
wg := new(sync.WaitGroup)
for _, remote := range ns.remoteServices {
wg.Add(1)
go func(r noncepb.NonceServiceClient) {
defer wg.Done()
resp, err := r.Redeem(subCtx, msg)
if err != nil {
ns.log.Errf("remote Redeem call failed: %s", err)
return
}
results <- *resp.Valid
}(remote)
}
go func() {
wg.Wait()
close(results)
}()
for result := range results {
select {
case <-subCtx.Done():
return false
default:
if result {
return true
}
}
}
return false
}
func (ns *nonceServer) Redeem(ctx context.Context, msg *noncepb.NonceMessage) (*noncepb.ValidMessage, error) {
if msg.Nonce == nil {
return nil, errors.New("Incomplete gRPC request message")
}
valid := ns.inner.Valid(*msg.Nonce)
// If the nonce was not valid, we have configured remote nonce services,
// and this Redeem message wasn't forwarded, then forward it to the
// remote services
if !valid && len(ns.remoteServices) > 0 && msg.Forwarded != nil && !*msg.Forwarded {
valid = ns.remoteRedeem(ctx, msg)
}
return &noncepb.ValidMessage{Valid: &valid}, nil
}
@ -41,6 +95,8 @@ func (ns *nonceServer) Nonce(_ context.Context, _ *corepb.Empty) (*noncepb.Nonce
}
func main() {
grpcAddr := flag.String("addr", "", "gRPC listen address override")
debugAddr := flag.String("debug-addr", "", "Debug server address override")
configFile := flag.String("config", "", "File path to the configuration file for this service")
flag.Parse()
@ -48,6 +104,13 @@ func main() {
err := cmd.ReadConfigFile(*configFile, &c)
cmd.FailOnError(err, "Reading JSON config file into config structure")
if *grpcAddr != "" {
c.NonceService.GRPC.Address = *grpcAddr
}
if *debugAddr != "" {
c.NonceService.DebugAddr = *debugAddr
}
scope, logger := cmd.StatsAndLogging(c.NonceService.Syslog, c.NonceService.DebugAddr)
defer logger.AuditPanic()
logger.Info(cmd.VersionString())
@ -57,10 +120,22 @@ func main() {
tlsConfig, err := c.NonceService.TLS.Load()
cmd.FailOnError(err, "tlsConfig config")
nonceServer := &nonceServer{inner: ns, log: logger}
if len(c.NonceService.RemoteNonceServices) > 0 {
clientMetrics := bgrpc.NewClientMetrics(scope)
clk := cmd.Clock()
for _, remoteNonceConfig := range c.NonceService.RemoteNonceServices {
rnsConn, err := bgrpc.ClientSetup(&remoteNonceConfig, tlsConfig, clientMetrics, clk)
cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to Nonce service")
nonceServer.remoteServices = append(nonceServer.remoteServices, noncepb.NewNonceServiceClient(rnsConn))
}
}
serverMetrics := bgrpc.NewServerMetrics(scope)
grpcSrv, l, err := bgrpc.NewServer(c.NonceService.GRPC, tlsConfig, serverMetrics, cmd.Clock())
cmd.FailOnError(err, "Unable to setup nonce service gRPC server")
noncepb.RegisterNonceServiceServer(grpcSrv, &nonceServer{inner: ns})
noncepb.RegisterNonceServiceServer(grpcSrv, nonceServer)
go cmd.CatchSignals(logger, grpcSrv.GracefulStop)

118
cmd/nonce-service/main_test.go Normal file
View File

@ -0,0 +1,118 @@
package main
import (
"context"
"errors"
"testing"
"time"
corepb "github.com/letsencrypt/boulder/core/proto"
blog "github.com/letsencrypt/boulder/log"
"github.com/letsencrypt/boulder/metrics"
"github.com/letsencrypt/boulder/nonce"
noncepb "github.com/letsencrypt/boulder/nonce/proto"
"github.com/letsencrypt/boulder/test"
"google.golang.org/grpc"
)
type workingRemote struct{ resp bool }
func (wr *workingRemote) Redeem(ctx context.Context, in *noncepb.NonceMessage, opts ...grpc.CallOption) (*noncepb.ValidMessage, error) {
return &noncepb.ValidMessage{
Valid: &wr.resp,
}, nil
}
func (wr *workingRemote) Nonce(ctx context.Context, in *corepb.Empty, opts ...grpc.CallOption) (*noncepb.NonceMessage, error) {
return nil, nil
}
type sleepingRemote struct{}
func (sr *sleepingRemote) Redeem(ctx context.Context, in *noncepb.NonceMessage, opts ...grpc.CallOption) (*noncepb.ValidMessage, error) {
time.Sleep(time.Millisecond * 50)
valid := true
return &noncepb.ValidMessage{
Valid: &valid,
}, nil
}
func (sr *sleepingRemote) Nonce(ctx context.Context, in *corepb.Empty, opts ...grpc.CallOption) (*noncepb.NonceMessage, error) {
return nil, nil
}
type brokenRemote struct{}
func (br *brokenRemote) Redeem(ctx context.Context, in *noncepb.NonceMessage, opts ...grpc.CallOption) (*noncepb.ValidMessage, error) {
return nil, errors.New("BROKE!")
}
func (br *brokenRemote) Nonce(ctx context.Context, in *corepb.Empty, opts ...grpc.CallOption) (*noncepb.NonceMessage, error) {
return nil, nil
}
func TestRemoteRedeem(t *testing.T) {
l := blog.NewMock()
innerNs, err := nonce.NewNonceService(metrics.NewNoopScope(), 1)
test.AssertNotError(t, err, "NewNonceService failed")
ns := nonceServer{log: l, inner: innerNs}
// Working remote returning valid nonce message
ns.remoteServices = []noncepb.NonceServiceClient{
&workingRemote{resp: false},
&workingRemote{resp: true},
}
nonce := "asd"
forwarded := false
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Hour))
resp, err := ns.Redeem(ctx, &noncepb.NonceMessage{Nonce: &nonce, Forwarded: &forwarded})
cancel()
test.AssertNotError(t, err, "Redeem failed")
test.Assert(t, *resp.Valid, "Redeem returned the wrong response")
// Working remotes returning invalid nonce message
ns.remoteServices = []noncepb.NonceServiceClient{
&workingRemote{resp: false},
&workingRemote{resp: false},
}
ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(time.Hour))
resp, err = ns.Redeem(ctx, &noncepb.NonceMessage{Nonce: &nonce, Forwarded: &forwarded})
cancel()
test.AssertNotError(t, err, "Redeem failed")
test.Assert(t, !*resp.Valid, "Redeem returned the wrong response")
// Sleeping remotes returns valid nonce message, but after 50ms, Redeem should return false
ns.remoteServices = []noncepb.NonceServiceClient{
&sleepingRemote{},
&sleepingRemote{},
}
ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(time.Millisecond))
resp, err = ns.Redeem(ctx, &noncepb.NonceMessage{Nonce: &nonce, Forwarded: &forwarded})
cancel()
test.AssertNotError(t, err, "Redeem failed")
test.Assert(t, !*resp.Valid, "Redeem returned the wrong response")
// Already forwarded message, Redeem should return false
ns.remoteServices = []noncepb.NonceServiceClient{
&workingRemote{resp: true},
&workingRemote{resp: true},
}
forwarded = true
ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(time.Hour))
resp, err = ns.Redeem(ctx, &noncepb.NonceMessage{Nonce: &nonce, Forwarded: &forwarded})
cancel()
test.AssertNotError(t, err, "Redeem failed")
test.Assert(t, !*resp.Valid, "Redeem returned the wrong response")
// Broken remotes, Redeem should return false
ns.remoteServices = []noncepb.NonceServiceClient{
&brokenRemote{},
&brokenRemote{},
}
ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(time.Millisecond))
resp, err = ns.Redeem(ctx, &noncepb.NonceMessage{Nonce: &nonce, Forwarded: &forwarded})
cancel()
test.AssertNotError(t, err, "Redeem failed")
test.Assert(t, !*resp.Valid, "Redeem returned the wrong response")
}

27
nonce/proto/nonce.pb.go
View File

@ -27,6 +27,7 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
type NonceMessage struct {
Nonce *string `protobuf:"bytes,1,opt,name=nonce" json:"nonce,omitempty"`
Forwarded *bool `protobuf:"varint,2,opt,name=forwarded" json:"forwarded,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -64,6 +65,13 @@ func (m *NonceMessage) GetNonce() string {
return ""
}
func (m *NonceMessage) GetForwarded() bool {
if m != nil && m.Forwarded != nil {
return *m.Forwarded
}
return false
}
type ValidMessage struct {
Valid *bool `protobuf:"varint,1,opt,name=valid" json:"valid,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -111,18 +119,19 @@ func init() {
func init() { proto.RegisterFile("nonce/proto/nonce.proto", fileDescriptor_9197b76ef104b424) }
var fileDescriptor_9197b76ef104b424 = []byte{
// 164 bytes of a gzipped FileDescriptorProto
// 189 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0xcf, 0xcb, 0xcf, 0x4b,
0x4e, 0xd5, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0xd7, 0x07, 0xb3, 0xf5, 0xc0, 0x6c, 0x21, 0x56, 0x30,
0x47, 0x4a, 0x34, 0x39, 0xbf, 0x08, 0x26, 0x0d, 0x62, 0x42, 0x64, 0x95, 0x54, 0xb8, 0x78, 0xfc,
0x47, 0x4a, 0x34, 0x39, 0xbf, 0x08, 0x26, 0x0d, 0x62, 0x42, 0x64, 0x95, 0x9c, 0xb8, 0x78, 0xfc,
0x40, 0xf2, 0xbe, 0xa9, 0xc5, 0xc5, 0x89, 0xe9, 0xa9, 0x42, 0x22, 0x5c, 0x10, 0xf5, 0x12, 0x8c,
0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x10, 0x0e, 0x48, 0x55, 0x58, 0x62, 0x4e, 0x66, 0x0a, 0x92, 0xaa,
0x32, 0x10, 0x1f, 0xac, 0x8a, 0x23, 0x08, 0xc2, 0x31, 0x2a, 0x84, 0x9a, 0x15, 0x9c, 0x5a, 0x54,
0x96, 0x99, 0x9c, 0x2a, 0xa4, 0xcd, 0xc5, 0x0a, 0xe6, 0x0b, 0x71, 0xeb, 0x81, 0x6d, 0x74, 0xcd,
0x2d, 0x28, 0xa9, 0x94, 0x12, 0xd6, 0x83, 0xb8, 0x0e, 0xd9, 0x5a, 0x25, 0x06, 0x21, 0x13, 0x2e,
0xb6, 0xa0, 0xd4, 0x94, 0xd4, 0xd4, 0x5c, 0x21, 0x6c, 0x0a, 0xe0, 0xba, 0x90, 0x9d, 0xa1, 0xc4,
0xe0, 0xc4, 0x1e, 0xc5, 0x0a, 0xf6, 0x07, 0x20, 0x00, 0x00, 0xff, 0xff, 0x38, 0x1f, 0x03, 0xab,
0xff, 0x00, 0x00, 0x00,
0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x10, 0x8e, 0x90, 0x0c, 0x17, 0x67, 0x5a, 0x7e, 0x51, 0x79, 0x62,
0x51, 0x4a, 0x6a, 0x8a, 0x04, 0x93, 0x02, 0xa3, 0x06, 0x47, 0x10, 0x42, 0x40, 0x49, 0x85, 0x8b,
0x27, 0x2c, 0x31, 0x27, 0x33, 0x05, 0xc9, 0x8c, 0x32, 0x10, 0x1f, 0x6c, 0x06, 0x47, 0x10, 0x84,
0x63, 0x54, 0x08, 0xb5, 0x29, 0x38, 0xb5, 0xa8, 0x2c, 0x33, 0x39, 0x55, 0x48, 0x9b, 0x8b, 0x15,
0xcc, 0x17, 0xe2, 0xd6, 0x03, 0xbb, 0xc7, 0x35, 0xb7, 0xa0, 0xa4, 0x52, 0x4a, 0x58, 0x0f, 0xe2,
0x76, 0x64, 0x47, 0x29, 0x31, 0x08, 0x99, 0x70, 0xb1, 0x05, 0xa5, 0xa6, 0xa4, 0xa6, 0xe6, 0x0a,
0x61, 0x53, 0x00, 0xd7, 0x85, 0xec, 0x0c, 0x25, 0x06, 0x27, 0xf6, 0x28, 0x56, 0xb0, 0x2f, 0x01,
0x01, 0x00, 0x00, 0xff, 0xff, 0x29, 0xf6, 0x52, 0xfd, 0x1d, 0x01, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.

1
nonce/proto/nonce.proto
View File

@ -12,6 +12,7 @@ service NonceService {
message NonceMessage {
optional string nonce = 1;
optional bool forwarded = 2;
}
message ValidMessage {

15
test/config-next/nonce.json
View File

@ -8,13 +8,24 @@
"grpc": {
"address": ":9101",
"clientNames": [
"wfe.boulder"
"wfe.boulder",
"nonce.boulder"
]
},
"tls": {
"caCertFile": "test/grpc-creds/minica.pem",
"certFile": "test/grpc-creds/nonce.boulder/cert.pem",
"keyFile": "test/grpc-creds/nonce.boulder/key.pem"
}
},
"remoteNonceServices": [
{
"serverAddress": "nonce.boulder:9102",
"timeout": "15s"
},
{
"serverAddress": "nonce.boulder:9101",
"timeout": "15s"
}
]
}
}

2
test/grpc-creds/generate.sh
View File

@ -10,7 +10,7 @@ command -v minica >/dev/null 2>&1 || {
}
for HOSTNAME in admin-revoker.boulder expiration-mailer.boulder \
ocsp-updater.boulder orphan-finder.boulder wfe.boulder akamai-purger.boulder ; do
ocsp-updater.boulder orphan-finder.boulder wfe.boulder akamai-purger.boulder nonce.boulder ; do
minica -domains ${HOSTNAME}
done

1
test/startservers.py
View File

@ -88,6 +88,7 @@ def start(race_detection, fakeclock=None, config_dir=default_config_dir):
[8002, './bin/boulder-ra --config %s --addr ra1.boulder:9094 --debug-addr :8002' % os.path.join(config_dir, "ra.json")],
[8102, './bin/boulder-ra --config %s --addr ra2.boulder:9094 --debug-addr :8102' % os.path.join(config_dir, "ra.json")],
[8111, './bin/nonce-service --config %s' % os.path.join(config_dir, "nonce.json")],
[8112, './bin/nonce-service --config %s --addr nonce.boulder:9102 --debug-addr :8112' % os.path.join(config_dir, "nonce.json")],
[4431, './bin/boulder-wfe2 --config %s' % os.path.join(config_dir, "wfe2.json")],
[4000, './bin/boulder-wfe --config %s' % os.path.join(config_dir, "wfe.json")],
])