letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

va: Make the primary VA aware of the Perspective and RIR of each remote (#7839) 

- Make the primary VA aware of the expected Perspective and RIR of each
remote VA.
- All Perspectives should be unique, have the primary VA check for
duplicate Perspectives at startup.
- Update test setup functions to ensure that each remote VA client and
corresponding inmem impl have a matching perspective and RIR.

Part of #7819
This commit is contained in:
Samantha Frank 2024-11-25 13:02:03 -05:00 committed by GitHub
parent 7791262815
commit c3948314ff
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 420 additions and 318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
cmd/boulder-va/main.go
View File

@ -15,10 +15,44 @@ import (
vapb "github.com/letsencrypt/boulder/va/proto"
)
// RemoteVAGRPCClientConfig contains the information necessary to setup a gRPC
// client connection. The following GRPC client configuration field combinations
// are allowed:
//
// ServerIPAddresses, [Timeout]
// ServerAddress, DNSAuthority, [Timeout], [HostOverride]
// SRVLookup, DNSAuthority, [Timeout], [HostOverride], [SRVResolver]
// SRVLookups, DNSAuthority, [Timeout], [HostOverride], [SRVResolver]
type RemoteVAGRPCClientConfig struct {
cmd.GRPCClientConfig
// Perspective uniquely identifies the Network Perspective used to
// perform the validation, as specified in BRs Section 5.4.1,
// Requirement 2.7 ("Multi-Perspective Issuance Corroboration attempts
// from each Network Perspective"). It should uniquely identify a group
// of RVAs deployed in the same datacenter.
//
// TODO(#7615): Make mandatory.
Perspective string `validate:"omitempty"`
// RIR indicates the Regional Internet Registry where this RVA is
// located. This field is used to identify the RIR region from which a
// given validation was performed, as specified in the "Phased
// Implementation Timeline" in BRs Section 3.2.2.9. It must be one of
// the following values:
// - ARIN
// - RIPE
// - APNIC
// - LACNIC
// - AfriNIC
//
// TODO(#7615): Make mandatory.
RIR string `validate:"omitempty,oneof=ARIN RIPE APNIC LACNIC AfriNIC"`
}
type Config struct {
VA struct {
vaConfig.Common
RemoteVAs []cmd.GRPCClientConfig `validate:"omitempty,dive"`
RemoteVAs []RemoteVAGRPCClientConfig `validate:"omitempty,dive"`
// Deprecated and ignored
MaxRemoteValidationFailures int `validate:"omitempty,min=0,required_with=RemoteVAs"`
Features features.Config
@ -92,7 +126,7 @@ func main() {
if len(c.VA.RemoteVAs) > 0 {
for _, rva := range c.VA.RemoteVAs {
rva := rva
vaConn, err := bgrpc.ClientSetup(&rva, tlsConfig, scope, clk)
vaConn, err := bgrpc.ClientSetup(&rva.GRPCClientConfig, tlsConfig, scope, clk)
cmd.FailOnError(err, "Unable to create remote VA client")
remotes = append(
remotes,
@ -101,7 +135,9 @@ func main() {
VAClient: vapb.NewVAClient(vaConn),
CAAClient: vapb.NewCAAClient(vaConn),
},
Address: rva.ServerAddress,
Address: rva.ServerAddress,
Perspective: rva.Perspective,
RIR: rva.RIR,
},
)
}

2
test/config-next/remoteva-a.json
View File

@ -37,7 +37,7 @@
"http://boulder.service.consul:4000/acme/reg/",
"http://boulder.service.consul:4001/acme/acct/"
],
"perspective": "development",
"perspective": "dadaist",
"rir": "ARIN"
},
"syslog": {

2
test/config-next/remoteva-b.json
View File

@ -37,7 +37,7 @@
"http://boulder.service.consul:4000/acme/reg/",
"http://boulder.service.consul:4001/acme/acct/"
],
"perspective": "development",
"perspective": "surrealist",
"rir": "RIPE"
},
"syslog": {

2
test/config-next/remoteva-c.json
View File

@ -37,7 +37,7 @@
"http://boulder.service.consul:4000/acme/reg/",
"http://boulder.service.consul:4001/acme/acct/"
],
"perspective": "development",
"perspective": "cubist",
"rir": "ARIN"
},
"syslog": {

12
test/config-next/va.json
View File

@ -46,17 +46,23 @@
{
"serverAddress": "rva1.service.consul:9397",
"timeout": "15s",
"hostOverride": "rva1.boulder"
"hostOverride": "rva1.boulder",
"perspective": "dadaist",
"rir": "ARIN"
},
{
"serverAddress": "rva1.service.consul:9498",
"timeout": "15s",
"hostOverride": "rva1.boulder"
"hostOverride": "rva1.boulder",
"perspective": "surrealist",
"rir": "RIPE"
},
{
"serverAddress": "rva1.service.consul:9499",
"timeout": "15s",
"hostOverride": "rva1.boulder"
"hostOverride": "rva1.boulder",
"perspective": "cubist",
"rir": "ARIN"
}
],
"accountURIPrefixes": [

169
va/caa_test.go
View File

@ -590,9 +590,8 @@ func (b caaBrokenDNS) LookupCAA(_ context.Context, domain string) ([]*dns.CAA, s
}
func TestDisabledMultiCAARechecking(t *testing.T) {
brokenRVA := setupRemote(nil, "broken", caaBrokenDNS{}, "", "")
remoteVAs := []RemoteVA{{brokenRVA, "broken"}}
va, _ := setup(nil, "local", remoteVAs, nil)
remoteVAs := []remoteConf{{ua: "broken", rir: arin, dns: caaBrokenDNS{}}}
va, _ := setupWithRemotes(nil, "local", remoteVAs, nil)
features.Set(features.Config{
EnforceMultiCAA: false,
@ -664,15 +663,11 @@ func TestMultiCAARechecking(t *testing.T) {
brokenUA = "broken"
hijackedUA = "hijacked"
)
remoteVA := setupRemote(nil, remoteUA, nil, "", "")
brokenVA := setupRemote(nil, brokenUA, caaBrokenDNS{}, "", "")
// Returns incorrect results
hijackedVA := setupRemote(nil, hijackedUA, caaHijackedDNS{}, "", "")
testCases := []struct {
name string
domains string
remoteVAs []RemoteVA
remoteVAs []remoteConf
expectedProbSubstring string
expectedProbType probs.ProblemType
expectedDiffLogSubstring string
@ -683,10 +678,10 @@ func TestMultiCAARechecking(t *testing.T) {
name: "all VAs functional, no CAA records",
domains: "present-dns-only.com",
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{remoteVA, remoteUA},
{remoteVA, remoteUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: remoteUA, rir: arin},
{ua: remoteUA, rir: ripe},
{ua: remoteUA, rir: apnic},
},
expectedLabels: prometheus.Labels{
"operation": opCAA,
@ -702,10 +697,10 @@ func TestMultiCAARechecking(t *testing.T) {
localDNSClient: caaBrokenDNS{},
expectedProbSubstring: "While processing CAA for present-dns-only.com: dnsClient is broken",
expectedProbType: probs.DNSProblem,
remoteVAs: []RemoteVA{
{remoteVA, remoteUA},
{remoteVA, remoteUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: remoteUA, rir: arin},
{ua: remoteUA, rir: ripe},
{ua: remoteUA, rir: apnic},
},
expectedLabels: prometheus.Labels{
"operation": opCAA,
@ -720,10 +715,10 @@ func TestMultiCAARechecking(t *testing.T) {
domains: "present-dns-only.com",
localDNSClient: caaMockDNS{},
expectedDiffLogSubstring: `RemoteSuccesses":2,"RemoteFailures":[{"VAHostname":"broken","Problem":{"type":"dns","detail":"While processing CAA for`,
remoteVAs: []RemoteVA{
{brokenVA, brokenUA},
{remoteVA, remoteUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: brokenUA, rir: arin, dns: caaBrokenDNS{}},
{ua: remoteUA, rir: ripe},
{ua: remoteUA, rir: apnic},
},
expectedLabels: prometheus.Labels{
"operation": opCAA,
@ -740,10 +735,10 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbType: probs.DNSProblem,
expectedDiffLogSubstring: `RemoteSuccesses":1,"RemoteFailures":[{"VAHostname":"broken","Problem":{"type":"dns","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{brokenVA, brokenUA},
{brokenVA, brokenUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: brokenUA, rir: arin, dns: caaBrokenDNS{}},
{ua: brokenUA, rir: ripe, dns: caaBrokenDNS{}},
{ua: remoteUA, rir: apnic},
},
expectedLabels: prometheus.Labels{
"operation": opCAA,
@ -760,10 +755,10 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbType: probs.DNSProblem,
expectedDiffLogSubstring: `RemoteSuccesses":0,"RemoteFailures":[{"VAHostname":"broken","Problem":{"type":"dns","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{brokenVA, brokenUA},
{brokenVA, brokenUA},
{brokenVA, brokenUA},
remoteVAs: []remoteConf{
{ua: brokenUA, rir: arin, dns: caaBrokenDNS{}},
{ua: brokenUA, rir: ripe, dns: caaBrokenDNS{}},
{ua: brokenUA, rir: apnic, dns: caaBrokenDNS{}},
},
expectedLabels: prometheus.Labels{
"operation": opCAA,
@ -777,10 +772,10 @@ func TestMultiCAARechecking(t *testing.T) {
name: "all VAs functional, CAA issue type present",
domains: "present.com",
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{remoteVA, remoteUA},
{remoteVA, remoteUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: remoteUA, rir: arin},
{ua: remoteUA, rir: ripe},
{ua: remoteUA, rir: apnic},
},
expectedLabels: prometheus.Labels{
"operation": opCAA,
@ -795,10 +790,10 @@ func TestMultiCAARechecking(t *testing.T) {
domains: "present.com",
expectedDiffLogSubstring: `RemoteSuccesses":2,"RemoteFailures":[{"VAHostname":"broken","Problem":{"type":"dns","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{brokenVA, brokenUA},
{remoteVA, remoteUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: brokenUA, rir: arin, dns: caaBrokenDNS{}},
{ua: remoteUA, rir: ripe},
{ua: remoteUA, rir: apnic},
},
expectedLabels: prometheus.Labels{
"operation": opCAA,
@ -815,10 +810,10 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbType: probs.DNSProblem,
expectedDiffLogSubstring: `RemoteSuccesses":1,"RemoteFailures":[{"VAHostname":"broken","Problem":{"type":"dns","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{brokenVA, brokenUA},
{brokenVA, brokenUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: brokenUA, rir: arin, dns: caaBrokenDNS{}},
{ua: brokenUA, rir: ripe, dns: caaBrokenDNS{}},
{ua: remoteUA, rir: apnic},
},
expectedLabels: prometheus.Labels{
"operation": opCAA,
@ -835,10 +830,10 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbType: probs.DNSProblem,
expectedDiffLogSubstring: `RemoteSuccesses":0,"RemoteFailures":[{"VAHostname":"broken","Problem":{"type":"dns","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{brokenVA, brokenUA},
{brokenVA, brokenUA},
{brokenVA, brokenUA},
remoteVAs: []remoteConf{
{ua: brokenUA, rir: arin, dns: caaBrokenDNS{}},
{ua: brokenUA, rir: ripe, dns: caaBrokenDNS{}},
{ua: brokenUA, rir: apnic, dns: caaBrokenDNS{}},
},
expectedLabels: prometheus.Labels{
"operation": opCAA,
@ -857,10 +852,10 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbSubstring: "CAA record for unsatisfiable.com prevents issuance",
expectedProbType: probs.CAAProblem,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{remoteVA, remoteUA},
{remoteVA, remoteUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: remoteUA, rir: arin},
{ua: remoteUA, rir: ripe},
{ua: remoteUA, rir: apnic},
},
},
{
@ -868,10 +863,10 @@ func TestMultiCAARechecking(t *testing.T) {
domains: "present.com",
expectedDiffLogSubstring: `RemoteSuccesses":2,"RemoteFailures":[{"VAHostname":"hijacked","Problem":{"type":"caa","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{hijackedVA, hijackedUA},
{remoteVA, remoteUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: hijackedUA, rir: arin, dns: caaHijackedDNS{}},
{ua: remoteUA, rir: ripe},
{ua: remoteUA, rir: apnic},
},
},
{
@ -881,10 +876,10 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbType: probs.CAAProblem,
expectedDiffLogSubstring: `RemoteSuccesses":1,"RemoteFailures":[{"VAHostname":"hijacked","Problem":{"type":"caa","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{hijackedVA, hijackedUA},
{hijackedVA, hijackedUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: hijackedUA, rir: arin, dns: caaHijackedDNS{}},
{ua: hijackedUA, rir: ripe, dns: caaHijackedDNS{}},
{ua: remoteUA, rir: apnic},
},
},
{
@ -894,10 +889,10 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbType: probs.CAAProblem,
expectedDiffLogSubstring: `RemoteSuccesses":0,"RemoteFailures":[{"VAHostname":"hijacked","Problem":{"type":"caa","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{hijackedVA, hijackedUA},
{hijackedVA, hijackedUA},
{hijackedVA, hijackedUA},
remoteVAs: []remoteConf{
{ua: hijackedUA, rir: arin, dns: caaHijackedDNS{}},
{ua: hijackedUA, rir: ripe, dns: caaHijackedDNS{}},
{ua: hijackedUA, rir: apnic, dns: caaHijackedDNS{}},
},
},
{
@ -905,10 +900,10 @@ func TestMultiCAARechecking(t *testing.T) {
domains: "satisfiable-wildcard.com",
expectedDiffLogSubstring: `RemoteSuccesses":2,"RemoteFailures":[{"VAHostname":"hijacked","Problem":{"type":"caa","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{hijackedVA, hijackedUA},
{remoteVA, remoteUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: hijackedUA, rir: arin, dns: caaHijackedDNS{}},
{ua: remoteUA, rir: ripe},
{ua: remoteUA, rir: apnic},
},
},
{
@ -918,10 +913,10 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbType: probs.CAAProblem,
expectedDiffLogSubstring: `RemoteSuccesses":1,"RemoteFailures":[{"VAHostname":"hijacked","Problem":{"type":"caa","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{hijackedVA, hijackedUA},
{hijackedVA, hijackedUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: hijackedUA, rir: arin, dns: caaHijackedDNS{}},
{ua: hijackedUA, rir: ripe, dns: caaHijackedDNS{}},
{ua: remoteUA, rir: apnic},
},
},
{
@ -931,10 +926,10 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbType: probs.CAAProblem,
expectedDiffLogSubstring: `RemoteSuccesses":0,"RemoteFailures":[{"VAHostname":"hijacked","Problem":{"type":"caa","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{hijackedVA, hijackedUA},
{hijackedVA, hijackedUA},
{hijackedVA, hijackedUA},
remoteVAs: []remoteConf{
{ua: hijackedUA, rir: arin, dns: caaHijackedDNS{}},
{ua: hijackedUA, rir: ripe, dns: caaHijackedDNS{}},
{ua: hijackedUA, rir: apnic, dns: caaHijackedDNS{}},
},
},
{
@ -942,10 +937,10 @@ func TestMultiCAARechecking(t *testing.T) {
domains: "satisfiable-wildcard.com",
expectedDiffLogSubstring: `RemoteSuccesses":2,"RemoteFailures":[{"VAHostname":"hijacked","Problem":{"type":"caa","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{hijackedVA, hijackedUA},
{remoteVA, remoteUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: hijackedUA, rir: arin, dns: caaHijackedDNS{}},
{ua: remoteUA, rir: ripe},
{ua: remoteUA, rir: apnic},
},
},
{
@ -955,10 +950,10 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbType: probs.CAAProblem,
expectedDiffLogSubstring: `RemoteSuccesses":1,"RemoteFailures":[{"VAHostname":"hijacked","Problem":{"type":"caa","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{hijackedVA, hijackedUA},
{hijackedVA, hijackedUA},
{remoteVA, remoteUA},
remoteVAs: []remoteConf{
{ua: hijackedUA, rir: arin, dns: caaHijackedDNS{}},
{ua: hijackedUA, rir: ripe, dns: caaHijackedDNS{}},
{ua: remoteUA, rir: apnic},
},
},
{
@ -968,17 +963,17 @@ func TestMultiCAARechecking(t *testing.T) {
expectedProbType: probs.CAAProblem,
expectedDiffLogSubstring: `RemoteSuccesses":0,"RemoteFailures":[{"VAHostname":"hijacked","Problem":{"type":"caa","detail":"While processing CAA for`,
localDNSClient: caaMockDNS{},
remoteVAs: []RemoteVA{
{hijackedVA, hijackedUA},
{hijackedVA, hijackedUA},
{hijackedVA, hijackedUA},
remoteVAs: []remoteConf{
{ua: hijackedUA, rir: arin, dns: caaHijackedDNS{}},
{ua: hijackedUA, rir: ripe, dns: caaHijackedDNS{}},
{ua: hijackedUA, rir: apnic, dns: caaHijackedDNS{}},
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
va, mockLog := setup(nil, localUA, tc.remoteVAs, tc.localDNSClient)
va, mockLog := setupWithRemotes(nil, localUA, tc.remoteVAs, tc.localDNSClient)
defer mockLog.Clear()
// MultiCAAFullResults: false is inherently flaky because of the
@ -1011,8 +1006,8 @@ func TestMultiCAARechecking(t *testing.T) {
}
var invalidRVACount int
for _, x := range va.remoteVAs {
if x.Address == "broken" || x.Address == "hijacked" {
for _, x := range tc.remoteVAs {
if x.ua == brokenUA || x.ua == hijackedUA {
invalidRVACount++
}
}

29
va/va.go
View File

@ -87,7 +87,9 @@ type RemoteClients struct {
// extract this metadata which is useful for debugging gRPC connection issues.
type RemoteVA struct {
RemoteClients
Address string
Address string
Perspective string
RIR string
}
type vaMetrics struct {
@ -235,6 +237,15 @@ func NewValidationAuthorityImpl(
return nil, errors.New("no account URI prefixes configured")
}
for i, va1 := range remoteVAs {
for j, va2 := range remoteVAs {
// TODO(#7819): Remove the != "" check once perspective is required.
if i != j && va1.Perspective == va2.Perspective && va1.Perspective != "" {
return nil, fmt.Errorf("duplicate remote VA perspective %q", va1.Perspective)
}
}
}
pc := newDefaultPortConfig()
va := &ValidationAuthorityImpl{
@ -456,9 +467,11 @@ func (va *ValidationAuthorityImpl) performRemoteValidation(
}
type response struct {
addr string
result *vapb.ValidationResult
err error
addr string
perspective string
rir string
result *vapb.ValidationResult
err error
}
subCtx, cancel := context.WithCancel(ctx)
@ -468,7 +481,7 @@ func (va *ValidationAuthorityImpl) performRemoteValidation(
for _, i := range rand.Perm(remoteVACount) {
go func(rva RemoteVA) {
res, err := rva.PerformValidation(subCtx, req)
responses <- &response{rva.Address, res, err}
responses <- &response{rva.Address, rva.Perspective, rva.RIR, res, err}
}(va.remoteVAs[i])
}
@ -482,7 +495,7 @@ func (va *ValidationAuthorityImpl) performRemoteValidation(
if resp.err != nil {
// Failed to communicate with the remote VA.
failed = append(failed, resp.addr)
failed = append(failed, resp.perspective)
if core.IsCanceled(resp.err) {
currProb = probs.ServerInternal("Secondary domain validation RPC canceled")
@ -492,7 +505,7 @@ func (va *ValidationAuthorityImpl) performRemoteValidation(
}
} else if resp.result.Problems != nil {
// The remote VA returned a problem.
failed = append(failed, resp.result.Perspective)
failed = append(failed, resp.perspective)
var err error
currProb, err = bgrpc.PBToProblemDetails(resp.result.Problems)
@ -502,7 +515,7 @@ func (va *ValidationAuthorityImpl) performRemoteValidation(
}
} else {
// The remote VA returned a successful result.
passed = append(passed, resp.result.Perspective)
passed = append(passed, resp.perspective)
}
if firstProb == nil && currProb != nil {

480
va/va_test.go
View File

@ -162,6 +162,67 @@ func setupRemote(srv *httptest.Server, userAgent string, mockDNSClientOverride b
return RemoteClients{VAClient: &inMemVA{*rva}, CAAClient: &inMemVA{*rva}}
}
// RIRs
const (
arin = "ARIN"
ripe = "RIPE"
apnic = "APNIC"
lacnic = "LACNIC"
afrinic = "AFRINIC"
)
// remoteConf is used in conjunction with setupRemotes/withRemotes to configure
// a remote VA.
type remoteConf struct {
// ua is optional, will default to "user agent 1.0". When set to "broken" or
// "hijacked", the Address field of the resulting RemoteVA will be set to
// match. This is a bit hacky, but it's the easiest way to satisfy some of
// our existing TestMultiCAARechecking tests.
ua string
// rir is required.
rir string
// dns is optional.
dns bdns.Client
// impl is optional.
impl RemoteClients
}
func setupRemotes(confs []remoteConf, srv *httptest.Server) []RemoteVA {
remoteVAs := make([]RemoteVA, 0, len(confs))
for i, c := range confs {
if c.rir == "" {
panic("rir is required")
}
// perspective MUST be unique for each remote VA, otherwise the VA will
// fail to start.
perspective := fmt.Sprintf("dc-%d-%s", i, c.rir)
clients := setupRemote(srv, c.ua, c.dns, perspective, c.rir)
if c.impl != (RemoteClients{}) {
clients = c.impl
}
var address string
if c.ua == "broken" {
address = "broken"
}
if c.ua == "hijacked" {
address = "hijacked"
}
remoteVAs = append(remoteVAs, RemoteVA{
Address: address,
RemoteClients: clients,
Perspective: perspective,
RIR: c.rir,
})
}
return remoteVAs
}
func setupWithRemotes(srv *httptest.Server, userAgent string, remotes []remoteConf, mockDNSClientOverride bdns.Client) (*ValidationAuthorityImpl, *blog.Mock) {
remoteVAs := setupRemotes(remotes, srv)
return setup(srv, userAgent, remoteVAs, mockDNSClientOverride)
}
type multiSrv struct {
*httptest.Server
@ -169,13 +230,10 @@ type multiSrv struct {
allowedUAs map[string]bool
}
func (s *multiSrv) setAllowedUAs(allowedUAs map[string]bool) {
s.mu.Lock()
defer s.mu.Unlock()
s.allowedUAs = allowedUAs
}
const slowRemoteSleepMillis = 1000
const (
slowUA = "slow remote"
slowRemoteSleepMillis = 1000
)
func httpMultiSrv(t *testing.T, token string, allowedUAs map[string]bool) *multiSrv {
t.Helper()
@ -185,7 +243,7 @@ func httpMultiSrv(t *testing.T, token string, allowedUAs map[string]bool) *multi
ms := &multiSrv{server, sync.Mutex{}, allowedUAs}
m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if r.UserAgent() == "slow remote" {
if r.UserAgent() == slowUA {
time.Sleep(slowRemoteSleepMillis)
}
ms.mu.Lock()
@ -248,6 +306,32 @@ func (inmem inMemVA) IsCAAValid(ctx context.Context, req *vapb.IsCAAValidRequest
return inmem.rva.IsCAAValid(ctx, req)
}
func TestNewValidationAuthorityImplWithDuplicateRemotes(t *testing.T) {
var remoteVAs []RemoteVA
for i := 0; i < 3; i++ {
remoteVAs = append(remoteVAs, RemoteVA{
RemoteClients: setupRemote(nil, "", nil, "dadaist", arin),
Perspective: "dadaist",
RIR: arin,
})
}
_, err := NewValidationAuthorityImpl(
&bdns.MockClient{Log: blog.NewMock()},
remoteVAs,
"user agent 1.0",
"letsencrypt.org",
metrics.NoopRegisterer,
clock.NewFake(),
blog.NewMock(),
accountURIPrefixes,
"example perspective",
"",
)
test.AssertError(t, err, "NewValidationAuthorityImpl allowed duplicate remote perspectives")
test.AssertContains(t, err.Error(), "duplicate remote VA perspective \"dadaist\"")
}
func TestValidateMalformedChallenge(t *testing.T) {
va, _ := setup(nil, "", nil, nil)
@ -366,37 +450,11 @@ func TestDCVAndCAASequencing(t *testing.T) {
}
func TestMultiVA(t *testing.T) {
t.Parallel()
// Create a new challenge to use for the httpSrv
req := createValidationRequest("localhost", core.ChallengeTypeHTTP01)
const (
remoteUA1 = "remote 1"
remoteUA2 = "remote 2"
remoteUA3 = "remote 3"
remoteUA4 = "remote 4"
localUA = "local 1"
)
allowedUAs := map[string]bool{
localUA: true,
remoteUA1: true,
remoteUA2: true,
remoteUA3: true,
remoteUA4: true,
}
// Create an IPv4 test server
ms := httpMultiSrv(t, expectedToken, allowedUAs)
defer ms.Close()
remoteVA1 := setupRemote(ms.Server, remoteUA1, nil, "", "")
remoteVA2 := setupRemote(ms.Server, remoteUA2, nil, "", "")
remoteVA3 := setupRemote(ms.Server, remoteUA3, nil, "", "")
remoteVA4 := setupRemote(ms.Server, remoteUA4, nil, "", "")
remoteVAs := []RemoteVA{
{remoteVA1, remoteUA1},
{remoteVA2, remoteUA2},
{remoteVA3, remoteUA3},
}
brokenVA := RemoteClients{
VAClient: brokenRemoteVA{},
CAAClient: brokenRemoteVA{},
@ -406,176 +464,187 @@ func TestMultiVA(t *testing.T) {
CAAClient: cancelledVA{},
}
unauthorized := probs.Unauthorized(fmt.Sprintf(
`The key authorization file from the server did not match this challenge. Expected %q (got "???")`,
expectedKeyAuthorization))
expectedInternalErrLine := fmt.Sprintf(
`ERR: \[AUDIT\] Remote VA "broken".PerformValidation failed: %s`,
errBrokenRemoteVA.Error())
testCases := []struct {
Name string
RemoteVAs []RemoteVA
AllowedUAs map[string]bool
ExpectedProb *probs.ProblemDetails
ExpectedLog string
Name string
Remotes []remoteConf
PrimaryUA string
ExpectedProbType string
ExpectedLogContains string
}{
{
// With local and all remote VAs working there should be no problem.
Name: "Local and remote VAs OK",
RemoteVAs: remoteVAs,
AllowedUAs: allowedUAs,
Name: "Local and remote VAs OK",
Remotes: []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe},
{ua: pass, rir: apnic},
},
PrimaryUA: pass,
},
{
// If the local VA fails everything should fail
Name: "Local VA bad, remote VAs OK",
RemoteVAs: remoteVAs,
AllowedUAs: map[string]bool{remoteUA1: true, remoteUA2: true},
ExpectedProb: unauthorized,
Name: "Local VA bad, remote VAs OK",
Remotes: []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe},
{ua: pass, rir: apnic},
},
PrimaryUA: fail,
ExpectedProbType: string(probs.UnauthorizedProblem),
},
{
// If one out of three remote VAs fails with an internal err it should succeed
Name: "Local VA ok, 1/3 remote VA internal err",
RemoteVAs: []RemoteVA{
{remoteVA1, remoteUA1},
{remoteVA2, remoteUA2},
{brokenVA, "broken"},
Remotes: []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe},
{ua: pass, rir: apnic, impl: brokenVA},
},
AllowedUAs: allowedUAs,
PrimaryUA: pass,
},
{
// If two out of three remote VAs fail with an internal err it should fail
Name: "Local VA ok, 2/3 remote VAs internal err",
RemoteVAs: []RemoteVA{
{remoteVA1, remoteUA1},
{brokenVA, "broken"},
{brokenVA, "broken"},
Remotes: []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe, impl: brokenVA},
{ua: pass, rir: apnic, impl: brokenVA},
},
AllowedUAs: allowedUAs,
ExpectedProb: probs.ServerInternal("During secondary domain validation: Secondary domain validation RPC failed"),
PrimaryUA: pass,
ExpectedProbType: string(probs.ServerInternalProblem),
// The real failure cause should be logged
ExpectedLog: expectedInternalErrLine,
ExpectedLogContains: errBrokenRemoteVA.Error(),
},
{
// If one out of five remote VAs fail with an internal err it should succeed
Name: "Local VA ok, 1/5 remote VAs internal err",
RemoteVAs: []RemoteVA{
{remoteVA1, remoteUA1},
{remoteVA2, remoteUA2},
{remoteVA3, remoteUA3},
{remoteVA4, remoteUA4},
{brokenVA, "broken"},
Remotes: []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe},
{ua: pass, rir: apnic},
{ua: pass, rir: lacnic},
{ua: pass, rir: afrinic, impl: brokenVA},
},
AllowedUAs: allowedUAs,
PrimaryUA: pass,
},
{
// If two out of five remote VAs fail with an internal err it should fail
Name: "Local VA ok, 2/5 remote VAs internal err",
RemoteVAs: []RemoteVA{
{remoteVA1, remoteUA1},
{remoteVA2, remoteUA2},
{remoteVA3, remoteUA3},
{brokenVA, "broken"},
{brokenVA, "broken"},
Remotes: []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe},
{ua: pass, rir: apnic},
{ua: pass, rir: arin, impl: brokenVA},
{ua: pass, rir: ripe, impl: brokenVA},
},
AllowedUAs: allowedUAs,
ExpectedProb: probs.ServerInternal("During secondary domain validation: Secondary domain validation RPC failed"),
PrimaryUA: pass,
ExpectedProbType: string(probs.ServerInternalProblem),
// The real failure cause should be logged
ExpectedLog: expectedInternalErrLine,
ExpectedLogContains: errBrokenRemoteVA.Error(),
},
{
// If two out of six remote VAs fail with an internal err it should succeed
Name: "Local VA ok, 2/6 remote VAs internal err",
RemoteVAs: []RemoteVA{
{remoteVA1, remoteUA1},
{remoteVA2, remoteUA2},
{remoteVA3, remoteUA3},
{remoteVA4, remoteUA4},
{brokenVA, "broken"},
{brokenVA, "broken"},
Remotes: []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe},
{ua: pass, rir: apnic},
{ua: pass, rir: lacnic},
{ua: pass, rir: afrinic, impl: brokenVA},
{ua: pass, rir: arin, impl: brokenVA},
},
AllowedUAs: allowedUAs,
PrimaryUA: pass,
},
{
// If three out of six remote VAs fail with an internal err it should fail
Name: "Local VA ok, 4/6 remote VAs internal err",
RemoteVAs: []RemoteVA{
{remoteVA1, remoteUA1},
{remoteVA2, remoteUA2},
{remoteVA3, remoteUA3},
{brokenVA, "broken"},
{brokenVA, "broken"},
{brokenVA, "broken"},
Remotes: []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe},
{ua: pass, rir: apnic},
{ua: pass, rir: lacnic, impl: brokenVA},
{ua: pass, rir: afrinic, impl: brokenVA},
{ua: pass, rir: arin, impl: brokenVA},
},
AllowedUAs: allowedUAs,
ExpectedProb: probs.ServerInternal("During secondary domain validation: Secondary domain validation RPC failed"),
PrimaryUA: pass,
ExpectedProbType: string(probs.ServerInternalProblem),
// The real failure cause should be logged
ExpectedLog: expectedInternalErrLine,
ExpectedLogContains: errBrokenRemoteVA.Error(),
},
{
// With only one working remote VA there should be a validation failure
Name: "Local VA and one remote VA OK",
RemoteVAs: remoteVAs,
AllowedUAs: map[string]bool{localUA: true, remoteUA2: true},
ExpectedProb: probs.Unauthorized(fmt.Sprintf(
`During secondary domain validation: The key authorization file from the server did not match this challenge. Expected %q (got "???")`,
expectedKeyAuthorization)),
Name: "Local VA and one remote VA OK",
Remotes: []remoteConf{
{ua: pass, rir: arin},
{ua: fail, rir: ripe},
{ua: fail, rir: apnic},
},
PrimaryUA: pass,
ExpectedProbType: string(probs.UnauthorizedProblem),
ExpectedLogContains: "During secondary domain validation: The key authorization file from the server",
},
{
// If one remote VA cancels, it should succeed
Name: "Local VA and one remote VA OK, one cancelled VA",
RemoteVAs: []RemoteVA{
{remoteVA1, remoteUA1},
{cancelledVA, remoteUA2},
{remoteVA3, remoteUA3},
Remotes: []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe, impl: cancelledVA},
{ua: pass, rir: apnic},
},
AllowedUAs: allowedUAs,
PrimaryUA: pass,
},
{
// If all remote VAs cancel, it should fail
Name: "Local VA OK, three cancelled remote VAs",
RemoteVAs: []RemoteVA{
{cancelledVA, remoteUA1},
{cancelledVA, remoteUA2},
{cancelledVA, remoteUA3},
Remotes: []remoteConf{
{ua: pass, rir: arin, impl: cancelledVA},
{ua: pass, rir: ripe, impl: cancelledVA},
{ua: pass, rir: apnic, impl: cancelledVA},
},
AllowedUAs: allowedUAs,
ExpectedProb: probs.ServerInternal("During secondary domain validation: Secondary domain validation RPC canceled"),
PrimaryUA: pass,
ExpectedProbType: string(probs.ServerInternalProblem),
ExpectedLogContains: "During secondary domain validation: Secondary domain validation RPC canceled",
},
{
// With the local and remote VAs seeing diff problems, we expect a problem.
Name: "Local and remote VA differential, full results, enforce multi VA",
RemoteVAs: remoteVAs,
AllowedUAs: map[string]bool{localUA: true},
ExpectedProb: probs.Unauthorized(fmt.Sprintf(
`During secondary domain validation: The key authorization file from the server did not match this challenge. Expected %q (got "???")`,
expectedKeyAuthorization)),
Name: "Local and remote VA differential, full results, enforce multi VA",
Remotes: []remoteConf{
{ua: fail, rir: arin},
{ua: fail, rir: ripe},
{ua: fail, rir: apnic},
},
PrimaryUA: pass,
ExpectedProbType: string(probs.UnauthorizedProblem),
ExpectedLogContains: "During secondary domain validation: The key authorization file from the server",
},
}
for _, tc := range testCases {
t.Run(tc.Name, func(t *testing.T) {
// Configure the test server with the testcase allowed UAs.
ms.setAllowedUAs(tc.AllowedUAs)
t.Parallel()
// Configure one test server per test case so that all tests can run in parallel.
ms := httpMultiSrv(t, expectedToken, map[string]bool{pass: true, fail: false})
defer ms.Close()
// Configure a primary VA with testcase remote VAs.
localVA, mockLog := setup(ms.Server, localUA, tc.RemoteVAs, nil)
localVA, mockLog := setupWithRemotes(ms.Server, tc.PrimaryUA, tc.Remotes, nil)
// Perform all validations
res, _ := localVA.PerformValidation(ctx, req)
if res.Problems == nil && tc.ExpectedProb != nil {
t.Errorf("expected prob %v, got nil", tc.ExpectedProb)
} else if res.Problems != nil && tc.ExpectedProb == nil {
if res.Problems == nil && tc.ExpectedProbType != "" {
t.Errorf("expected prob %v, got nil", tc.ExpectedProbType)
} else if res.Problems != nil && tc.ExpectedProbType == "" {
t.Errorf("expected no prob, got %v", res.Problems)
} else if res.Problems != nil && tc.ExpectedProb != nil {
} else if res.Problems != nil && tc.ExpectedProbType != "" {
// That result should match expected.
test.AssertEquals(t, res.Problems.ProblemType, string(tc.ExpectedProb.Type))
test.AssertEquals(t, res.Problems.Detail, tc.ExpectedProb.Detail)
test.AssertEquals(t, res.Problems.ProblemType, tc.ExpectedProbType)
}
if tc.ExpectedLog != "" {
lines := mockLog.GetAllMatching(tc.ExpectedLog)
if tc.ExpectedLogContains != "" {
lines := mockLog.GetAllMatching(tc.ExpectedLogContains)
if len(lines) == 0 {
t.Fatalf("Got log %v; expected %q", mockLog.GetAll(), tc.ExpectedLog)
t.Fatalf("Got log %v; expected %q", mockLog.GetAll(), tc.ExpectedLogContains)
}
}
})
@ -583,39 +652,48 @@ func TestMultiVA(t *testing.T) {
}
func TestMultiVAEarlyReturn(t *testing.T) {
const passUA = "pass"
const failUA = "fail"
// httpMultiSrv handles this specially by being slow
const slowRemoteUA = "slow remote"
allowedUAs := map[string]bool{
passUA: true,
}
ms := httpMultiSrv(t, expectedToken, allowedUAs)
defer ms.Close()
makeRemotes := func(userAgent ...string) []RemoteVA {
var rvas []RemoteVA
for i, ua := range userAgent {
clients := setupRemote(ms.Server, ua, nil, "", "")
rva := RemoteVA{clients, fmt.Sprintf("remote VA %d hostname", i)}
rvas = append(rvas, rva)
}
return rvas
}
t.Parallel()
testCases := []struct {
remoteUserAgents []string
remoteConfs []remoteConf
}{
{remoteUserAgents: []string{slowRemoteUA, passUA, failUA}},
{remoteUserAgents: []string{slowRemoteUA, slowRemoteUA, passUA, passUA, failUA}},
{remoteUserAgents: []string{slowRemoteUA, slowRemoteUA, passUA, passUA, failUA, failUA}},
{
remoteConfs: []remoteConf{
{ua: slowUA, rir: arin},
{ua: pass, rir: ripe},
{ua: fail, rir: apnic},
},
},
{
remoteConfs: []remoteConf{
{ua: slowUA, rir: arin},
{ua: slowUA, rir: ripe},
{ua: pass, rir: apnic},
{ua: pass, rir: arin},
{ua: fail, rir: ripe},
},
},
{
remoteConfs: []remoteConf{
{ua: slowUA, rir: arin},
{ua: slowUA, rir: ripe},
{ua: pass, rir: apnic},
{ua: pass, rir: arin},
{ua: fail, rir: ripe},
{ua: fail, rir: apnic},
},
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
rvas := makeRemotes(tc.remoteUserAgents...)
localVA, _ := setup(ms.Server, pass, rvas, nil)
t.Parallel()
// Configure one test server per test case so that all tests can run in parallel.
ms := httpMultiSrv(t, expectedToken, map[string]bool{pass: true, fail: false})
defer ms.Close()
localVA, _ := setupWithRemotes(ms.Server, pass, tc.remoteConfs, nil)
// Perform all validations
start := time.Now()
@ -643,35 +721,19 @@ func TestMultiVAEarlyReturn(t *testing.T) {
}
func TestMultiVAPolicy(t *testing.T) {
const (
remoteUA1 = "remote 1"
remoteUA2 = "remote 2"
remoteUA3 = "remote 3"
localUA = "local 1"
)
// Forbid all remote UAs to ensure that multi-va fails
allowedUAs := map[string]bool{
localUA: true,
remoteUA1: false,
remoteUA2: false,
remoteUA3: false,
}
t.Parallel()
ms := httpMultiSrv(t, expectedToken, allowedUAs)
ms := httpMultiSrv(t, expectedToken, map[string]bool{pass: true, fail: false})
defer ms.Close()
remoteVA1 := setupRemote(ms.Server, remoteUA1, nil, "", "")
remoteVA2 := setupRemote(ms.Server, remoteUA2, nil, "", "")
remoteVA3 := setupRemote(ms.Server, remoteUA3, nil, "", "")
remoteVAs := []RemoteVA{
{remoteVA1, remoteUA1},
{remoteVA2, remoteUA2},
{remoteVA3, remoteUA3},
remoteConfs := []remoteConf{
{ua: fail, rir: arin},
{ua: fail, rir: ripe},
{ua: fail, rir: apnic},
}
// Create a local test VA with the two remote VAs
localVA, _ := setup(ms.Server, localUA, remoteVAs, nil)
// Create a local test VA with the remote VAs
localVA, _ := setupWithRemotes(ms.Server, pass, remoteConfs, nil)
// Perform validation for a domain not in the disabledDomains list
req := createValidationRequest("letsencrypt.org", core.ChallengeTypeHTTP01)
@ -681,28 +743,19 @@ func TestMultiVAPolicy(t *testing.T) {
t.Error("expected prob from PerformValidation, got nil")
}
}
func TestMultiVALogging(t *testing.T) {
const (
rva1UA = "remote 1"
rva2UA = "remote 2"
rva3UA = "remote 3"
localUA = "local 1"
)
t.Parallel()
ms := httpMultiSrv(t, expectedToken, map[string]bool{localUA: true, rva1UA: true, rva2UA: true})
ms := httpMultiSrv(t, expectedToken, map[string]bool{pass: true, fail: false})
defer ms.Close()
rva1 := setupRemote(ms.Server, rva1UA, nil, "dev-arin", "ARIN")
rva2 := setupRemote(ms.Server, rva2UA, nil, "dev-ripe", "RIPE")
rva3 := setupRemote(ms.Server, rva3UA, nil, "dev-ripe", "RIPE")
remoteVAs := []RemoteVA{
{rva1, rva1UA},
{rva2, rva2UA},
{rva3, rva3UA},
remoteConfs := []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe},
{ua: pass, rir: apnic},
}
va, _ := setup(ms.Server, localUA, remoteVAs, nil)
va, _ := setupWithRemotes(ms.Server, pass, remoteConfs, nil)
req := createValidationRequest("letsencrypt.org", core.ChallengeTypeHTTP01)
res, err := va.PerformValidation(ctx, req)
test.Assert(t, res.Problems == nil, fmt.Sprintf("validation failed with: %#v", res.Problems))
@ -762,19 +815,12 @@ func TestDetailedError(t *testing.T) {
}
func TestLogRemoteDifferentials(t *testing.T) {
// Create some remote VAs
remoteVA1 := setupRemote(nil, "remote 1", nil, "", "")
remoteVA2 := setupRemote(nil, "remote 2", nil, "", "")
remoteVA3 := setupRemote(nil, "remote 3", nil, "", "")
// The VA will allow a max of 1 remote failure based on MPIC.
remoteVAs := []RemoteVA{
{remoteVA1, "remote 1"},
{remoteVA2, "remote 2"},
{remoteVA3, "remote 3"},
remoteConfs := []remoteConf{
{ua: pass, rir: arin},
{ua: pass, rir: ripe},
{ua: pass, rir: apnic},
}
localVA, mockLog := setup(nil, "local 1", remoteVAs, nil)
egProbA := probs.DNS("root DNS servers closed at 4:30pm")
egProbB := probs.OrderNotReady("please take a number")
@ -813,7 +859,13 @@ func TestLogRemoteDifferentials(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
mockLog.Clear()
t.Parallel()
// Configure one test server per test case so that all tests can run in parallel.
ms := httpMultiSrv(t, expectedToken, map[string]bool{pass: true, fail: false})
defer ms.Close()
localVA, mockLog := setupWithRemotes(ms.Server, pass, remoteConfs, nil)
localVA.logRemoteResults(
"example.com", 1999, "blorpus-01", tc.remoteProbs)