letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

va: include hostname in remote VA differentials. (#4411) 

Also rename the `RemoteVA.Addresses` field. The address is always
a singular value.
This commit is contained in:
Daniel McCarney 2019-08-30 13:32:44 -04:00 committed by GitHub
parent fe23dabd69
commit d67d76388c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 70 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/boulder-va/main.go
View File

@ -134,7 +134,7 @@ func main() {
remotes, remotes,
va.RemoteVA{ va.RemoteVA{
ValidationAuthority: bgrpc.NewValidationAuthorityGRPCClient(vaConn), ValidationAuthority: bgrpc.NewValidationAuthorityGRPCClient(vaConn),
Addresses: rva.ServerAddress, Address: rva.ServerAddress,
}, },
) )
} }

80
va/va.go
View File

@ -65,12 +65,12 @@ var (
h2SettingsFrameErrRegex = regexp.MustCompile(`(?:net\/http\: HTTP\/1\.x transport connection broken: )?malformed HTTP response \"\\x00\\x00\\x[a-f0-9]{2}\\x04\\x00\\x00\\x00\\x00\\x00.*"`) h2SettingsFrameErrRegex = regexp.MustCompile(`(?:net\/http\: HTTP\/1\.x transport connection broken: )?malformed HTTP response \"\\x00\\x00\\x[a-f0-9]{2}\\x04\\x00\\x00\\x00\\x00\\x00.*"`)
) )
// RemoteVA wraps the core.ValidationAuthority interface and adds a field containing the addresses // RemoteVA wraps the core.ValidationAuthority interface and adds a field containing the address
// of the remote gRPC server since the interface (and the underlying gRPC client) doesn't // of the remote gRPC server since the interface (and the underlying gRPC client) doesn't
// provide a way to extract this metadata which is useful for debugging gRPC connection issues. // provide a way to extract this metadata which is useful for debugging gRPC connection issues.
type RemoteVA struct { type RemoteVA struct {
core.ValidationAuthority core.ValidationAuthority
Addresses string Address string
} }
type vaMetrics struct { type vaMetrics struct {
@ -353,7 +353,7 @@ func (va *ValidationAuthorityImpl) performRemoteValidation(
domain string, domain string,
challenge core.Challenge, challenge core.Challenge,
authz core.Authorization, authz core.Authorization,
results chan *probs.ProblemDetails) { results chan *remoteValidationResult) {
for _, i := range rand.Perm(len(va.remoteVAs)) { for _, i := range rand.Perm(len(va.remoteVAs)) {
remoteVA := va.remoteVAs[i] remoteVA := va.remoteVAs[i]
go func(rva RemoteVA, index int) { go func(rva RemoteVA, index int) {
@ -368,7 +368,7 @@ func (va *ValidationAuthorityImpl) performRemoteValidation(
// If the non-nil err was a non-nil *probs.ProblemDetails then we can // If the non-nil err was a non-nil *probs.ProblemDetails then we can
// log it at an info level. It's a normal non-success validation // log it at an info level. It's a normal non-success validation
// result and the remote VA will have logged more detail. // result and the remote VA will have logged more detail.
va.log.Infof("Remote VA %q.PerformValidation returned problem: %s", rva.Addresses, err) va.log.Infof("Remote VA %q.PerformValidation returned problem: %s", rva.Address, err)
} else if ok && p == (*probs.ProblemDetails)(nil) { } else if ok && p == (*probs.ProblemDetails)(nil) {
// If the non-nil err was a nil *probs.ProblemDetails then we don't need to do // If the non-nil err was a nil *probs.ProblemDetails then we don't need to do
// anything. There isn't really an error here. // anything. There isn't really an error here.
@ -384,15 +384,20 @@ func (va *ValidationAuthorityImpl) performRemoteValidation(
// will later be returned as a server internal error // will later be returned as a server internal error
// without detail if the number of errors is >= va.maxRemoteFailures. // without detail if the number of errors is >= va.maxRemoteFailures.
// Log it at the error level so we can debug from logs. // Log it at the error level so we can debug from logs.
va.log.Errf("Remote VA %q.PerformValidation failed: %s", rva.Addresses, err) va.log.Errf("Remote VA %q.PerformValidation failed: %s", rva.Address, err)
} }
} }
result := &remoteValidationResult{
VAHostname: rva.Address,
}
if err == nil { if err == nil {
results <- nil results <- result
} else if prob, ok := err.(*probs.ProblemDetails); ok { } else if prob, ok := err.(*probs.ProblemDetails); ok {
results <- prob result.Problem = prob
results <- result
} else { } else {
results <- probs.ServerInternal("Remote PerformValidation RPC failed") result.Problem = probs.ServerInternal("Remote PerformValidation RPC failed")
results <- result
} }
}(remoteVA, i) }(remoteVA, i)
} }
@ -418,7 +423,7 @@ func (va *ValidationAuthorityImpl) processRemoteResults(
domain string, domain string,
challengeType string, challengeType string,
primaryResult *probs.ProblemDetails, primaryResult *probs.ProblemDetails,
remoteErrors chan *probs.ProblemDetails, remoteResultsChan chan *remoteValidationResult,
numRemoteVAs int) *probs.ProblemDetails { numRemoteVAs int) *probs.ProblemDetails {
state := "failure" state := "failure"
@ -435,15 +440,15 @@ func (va *ValidationAuthorityImpl) processRemoteResults(
good := 0 good := 0
bad := 0 bad := 0
var remoteProbs []*probs.ProblemDetails var remoteResults []*remoteValidationResult
var firstProb *probs.ProblemDetails var firstProb *probs.ProblemDetails
// Due to channel behavior this could block indefinitely and we rely on gRPC // Due to channel behavior this could block indefinitely and we rely on gRPC
// honoring the context deadline used in client calls to prevent that from // honoring the context deadline used in client calls to prevent that from
// happening. // happening.
for prob := range remoteErrors { for result := range remoteResultsChan {
// Add the problem to the slice // Add the result to the slice
remoteProbs = append(remoteProbs, prob) remoteResults = append(remoteResults, result)
if prob == nil { if result.Problem == nil {
good++ good++
} else { } else {
bad++ bad++
@ -451,8 +456,8 @@ func (va *ValidationAuthorityImpl) processRemoteResults(
// Store the first non-nil problem to return later (if `MultiVAFullResults` // Store the first non-nil problem to return later (if `MultiVAFullResults`
// is enabled). // is enabled).
if firstProb == nil && prob != nil { if firstProb == nil && result.Problem != nil {
firstProb = prob firstProb = result.Problem
} }
// If MultiVAFullResults isn't enabled then return early whenever the // If MultiVAFullResults isn't enabled then return early whenever the
@ -462,13 +467,13 @@ func (va *ValidationAuthorityImpl) processRemoteResults(
state = "success" state = "success"
return nil return nil
} else if bad > va.maxRemoteFailures { } else if bad > va.maxRemoteFailures {
return prob return result.Problem
} }
} }
// If we haven't returned early because of MultiVAFullResults being enabled // If we haven't returned early because of MultiVAFullResults being enabled
// we need to break the loop once all of the VAs have returned a result. // we need to break the loop once all of the VAs have returned a result.
if len(remoteProbs) == numRemoteVAs { if len(remoteResults) == numRemoteVAs {
break break
} }
} }
@ -476,7 +481,7 @@ func (va *ValidationAuthorityImpl) processRemoteResults(
// If we are using `features.MultiVAFullResults` then we haven't returned // If we are using `features.MultiVAFullResults` then we haven't returned
// early and can now log the differential between what the primary VA saw and // early and can now log the differential between what the primary VA saw and
// what all of the remote VAs saw. // what all of the remote VAs saw.
va.logRemoteValidationDifferentials(domain, challengeType, primaryResult, remoteProbs) va.logRemoteValidationDifferentials(domain, challengeType, primaryResult, remoteResults)
// Based on the threshold of good/bad return nil or a problem. // Based on the threshold of good/bad return nil or a problem.
if good >= required { if good >= required {
@ -498,20 +503,20 @@ func (va *ValidationAuthorityImpl) logRemoteValidationDifferentials(
domain string, domain string,
challengeType string, challengeType string,
primaryResult *probs.ProblemDetails, primaryResult *probs.ProblemDetails,
remoteProbs []*probs.ProblemDetails) { remoteResults []*remoteValidationResult) {
var successes []*probs.ProblemDetails var successes []*remoteValidationResult
var failures []*probs.ProblemDetails var failures []*remoteValidationResult
allEqual := true allEqual := true
for _, e := range remoteProbs { for _, result := range remoteResults {
if e != primaryResult { if result.Problem != primaryResult {
allEqual = false allEqual = false
} }
if e == nil { if result.Problem == nil {
successes = append(successes, nil) successes = append(successes, result)
} else { } else {
failures = append(failures, e) failures = append(failures, result)
} }
} }
if allEqual { if allEqual {
@ -532,7 +537,7 @@ func (va *ValidationAuthorityImpl) logRemoteValidationDifferentials(
ChallengeType string ChallengeType string
PrimaryResult *probs.ProblemDetails PrimaryResult *probs.ProblemDetails
RemoteSuccesses int RemoteSuccesses int
RemoteFailures []*probs.ProblemDetails RemoteFailures []*remoteValidationResult
}{ }{
Domain: domain, Domain: domain,
ChallengeType: challengeType, ChallengeType: challengeType,
@ -554,6 +559,13 @@ func (va *ValidationAuthorityImpl) logRemoteValidationDifferentials(
va.log.Infof("remoteVADifferentials JSON=%s", string(logJSON)) va.log.Infof("remoteVADifferentials JSON=%s", string(logJSON))
} }
// remoteValidationResult is a struct that combines a problem details instance
// (that may be nil) with the remote VA hostname that produced it.
type remoteValidationResult struct {
VAHostname string
Problem *probs.ProblemDetails
}
// PerformValidation validates the given challenge. It always returns a list of // PerformValidation validates the given challenge. It always returns a list of
// validation records, even when it also returns an error. // validation records, even when it also returns an error.
func (va *ValidationAuthorityImpl) PerformValidation(ctx context.Context, domain string, challenge core.Challenge, authz core.Authorization) ([]core.ValidationRecord, error) { func (va *ValidationAuthorityImpl) PerformValidation(ctx context.Context, domain string, challenge core.Challenge, authz core.Authorization) ([]core.ValidationRecord, error) {
@ -564,10 +576,10 @@ func (va *ValidationAuthorityImpl) PerformValidation(ctx context.Context, domain
} }
vStart := va.clk.Now() vStart := va.clk.Now()
var remoteProbs chan *probs.ProblemDetails var remoteResults chan *remoteValidationResult
if remoteVACount := len(va.remoteVAs); remoteVACount > 0 { if remoteVACount := len(va.remoteVAs); remoteVACount > 0 {
remoteProbs = make(chan *probs.ProblemDetails, remoteVACount) remoteResults = make(chan *remoteValidationResult, remoteVACount)
go va.performRemoteValidation(ctx, domain, challenge, authz, remoteProbs) go va.performRemoteValidation(ctx, domain, challenge, authz, remoteResults)
} }
records, prob := va.validate(ctx, identifier.DNSIdentifier(domain), challenge, authz) records, prob := va.validate(ctx, identifier.DNSIdentifier(domain), challenge, authz)
@ -584,20 +596,20 @@ func (va *ValidationAuthorityImpl) PerformValidation(ctx context.Context, domain
challenge.Status = core.StatusInvalid challenge.Status = core.StatusInvalid
challenge.Error = prob challenge.Error = prob
logEvent.Error = prob.Error() logEvent.Error = prob.Error()
} else if remoteProbs != nil { } else if remoteResults != nil {
if !features.Enabled(features.EnforceMultiVA) && features.Enabled(features.MultiVAFullResults) { if !features.Enabled(features.EnforceMultiVA) && features.Enabled(features.MultiVAFullResults) {
// If we're not going to enforce multi VA but we are logging the // If we're not going to enforce multi VA but we are logging the
// differentials then collect and log the remote results in a separate go // differentials then collect and log the remote results in a separate go
// routine to avoid blocking the primary VA. // routine to avoid blocking the primary VA.
go func() { go func() {
_ = va.processRemoteResults(domain, string(challenge.Type), prob, remoteProbs, len(va.remoteVAs)) _ = va.processRemoteResults(domain, string(challenge.Type), prob, remoteResults, len(va.remoteVAs))
}() }()
// Since prob was nil and we're not enforcing the results from // Since prob was nil and we're not enforcing the results from
// `processRemoteResults` set the challenge status to valid so the // `processRemoteResults` set the challenge status to valid so the
// validationTime metrics increment has the correct result label. // validationTime metrics increment has the correct result label.
challenge.Status = core.StatusValid challenge.Status = core.StatusValid
} else if features.Enabled(features.EnforceMultiVA) { } else if features.Enabled(features.EnforceMultiVA) {
remoteProb := va.processRemoteResults(domain, string(challenge.Type), prob, remoteProbs, len(va.remoteVAs)) remoteProb := va.processRemoteResults(domain, string(challenge.Type), prob, remoteResults, len(va.remoteVAs))
if remoteProb != nil { if remoteProb != nil {
prob = remoteProb prob = remoteProb
challenge.Status = core.StatusInvalid challenge.Status = core.StatusInvalid

30
va/va_test.go
View File

@ -632,30 +632,46 @@ func TestLogRemoteValidationDifferentials(t *testing.T) {
testCases := []struct { testCases := []struct {
name string name string
primaryResult *probs.ProblemDetails primaryResult *probs.ProblemDetails
remoteProbs []*probs.ProblemDetails remoteProbs []*remoteValidationResult
expectedLog string expectedLog string
}{ }{
{ {
name: "remote and primary results equal (all nil)", name: "remote and primary results equal (all nil)",
primaryResult: nil, primaryResult: nil,
remoteProbs: nil, remoteProbs: []*remoteValidationResult{
&remoteValidationResult{Problem: nil, VAHostname: "remoteA"},
&remoteValidationResult{Problem: nil, VAHostname: "remoteB"},
&remoteValidationResult{Problem: nil, VAHostname: "remoteC"},
},
}, },
{ {
name: "remote and primary results equal (not nil)", name: "remote and primary results equal (not nil)",
primaryResult: egProbA, primaryResult: egProbA,
remoteProbs: []*probs.ProblemDetails{egProbA, egProbA, egProbA}, remoteProbs: []*remoteValidationResult{
&remoteValidationResult{Problem: egProbA, VAHostname: "remoteA"},
&remoteValidationResult{Problem: egProbA, VAHostname: "remoteB"},
&remoteValidationResult{Problem: egProbA, VAHostname: "remoteC"},
},
}, },
{ {
name: "remote and primary differ (primary nil)", name: "remote and primary differ (primary nil)",
primaryResult: nil, primaryResult: nil,
remoteProbs: []*probs.ProblemDetails{egProbA, nil, egProbB}, remoteProbs: []*remoteValidationResult{
expectedLog: `INFO: remoteVADifferentials JSON={"Domain":"example.com","ChallengeType":"blorpus-01","PrimaryResult":null,"RemoteSuccesses":1,"RemoteFailures":[{"type":"dns","detail":"root DNS servers closed at 4:30pm","status":400},{"type":"orderNotReady","detail":"please take a number","status":403}]}`, &remoteValidationResult{Problem: egProbA, VAHostname: "remoteA"},
&remoteValidationResult{Problem: nil, VAHostname: "remoteB"},
&remoteValidationResult{Problem: egProbB, VAHostname: "remoteC"},
},
expectedLog: `INFO: remoteVADifferentials JSON={"Domain":"example.com","ChallengeType":"blorpus-01","PrimaryResult":null,"RemoteSuccesses":1,"RemoteFailures":[{"VAHostname":"remoteA","Problem":{"type":"dns","detail":"root DNS servers closed at 4:30pm","status":400}},{"VAHostname":"remoteC","Problem":{"type":"orderNotReady","detail":"please take a number","status":403}}]}`,
}, },
{ {
name: "remote and primary differ (primary not nil)", name: "remote and primary differ (primary not nil)",
primaryResult: egProbA, primaryResult: egProbA,
remoteProbs: []*probs.ProblemDetails{nil, egProbB, nil}, remoteProbs: []*remoteValidationResult{
expectedLog: `INFO: remoteVADifferentials JSON={"Domain":"example.com","ChallengeType":"blorpus-01","PrimaryResult":{"type":"dns","detail":"root DNS servers closed at 4:30pm","status":400},"RemoteSuccesses":2,"RemoteFailures":[{"type":"orderNotReady","detail":"please take a number","status":403}]}`, &remoteValidationResult{Problem: nil, VAHostname: "remoteA"},
&remoteValidationResult{Problem: egProbB, VAHostname: "remoteB"},
&remoteValidationResult{Problem: nil, VAHostname: "remoteC"},
},
expectedLog: `INFO: remoteVADifferentials JSON={"Domain":"example.com","ChallengeType":"blorpus-01","PrimaryResult":{"type":"dns","detail":"root DNS servers closed at 4:30pm","status":400},"RemoteSuccesses":2,"RemoteFailures":[{"VAHostname":"remoteB","Problem":{"type":"orderNotReady","detail":"please take a number","status":403}}]}`,
}, },
} }