add retries and context deadlines to DNSResolver

This provides a means to add retries to DNS look ups, and, with some
future work, end retries early if our request deadline is blown. That
future work is tagged with #1292.

Updates #1258

Godeps/Godeps.json

@@ -1,6 +1,6 @@
 {
 	"ImportPath": "github.com/letsencrypt/boulder",
-	"GoVersion": "go1.5.1",
+	"GoVersion": "go1.5.2",
 	"Packages": [
 		"./..."
 	],
@@ -141,6 +141,9 @@
 			"Rev": "beef0f4390813b96e8e68fd78570396d0f4751fc"
 		},
 		{
+			"ImportPath": "golang.org/x/net/context",
+			"Rev": "ce84af2e5bf21582345e478b116afc7d4efaba3d"
+		},
 			"ImportPath": "gopkg.in/gorp.v1",
 			"Comment": "v1.7.1",
 			"Rev": "c87af80f3cc5036b55b83d77171e156791085e2e"

Godeps/_workspace/src/golang.org/x/net/context/context.go

@@ -0,0 +1,447 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package context defines the Context type, which carries deadlines,
+// cancelation signals, and other request-scoped values across API boundaries
+// and between processes.
+//
+// Incoming requests to a server should create a Context, and outgoing calls to
+// servers should accept a Context.  The chain of function calls between must
+// propagate the Context, optionally replacing it with a modified copy created
+// using WithDeadline, WithTimeout, WithCancel, or WithValue.
+//
+// Programs that use Contexts should follow these rules to keep interfaces
+// consistent across packages and enable static analysis tools to check context
+// propagation:
+//
+// Do not store Contexts inside a struct type; instead, pass a Context
+// explicitly to each function that needs it.  The Context should be the first
+// parameter, typically named ctx:
+//
+// 	func DoSomething(ctx context.Context, arg Arg) error {
+// 		// ... use ctx ...
+// 	}
+//
+// Do not pass a nil Context, even if a function permits it.  Pass context.TODO
+// if you are unsure about which Context to use.
+//
+// Use context Values only for request-scoped data that transits processes and
+// APIs, not for passing optional parameters to functions.
+//
+// The same Context may be passed to functions running in different goroutines;
+// Contexts are safe for simultaneous use by multiple goroutines.
+//
+// See http://blog.golang.org/context for example code for a server that uses
+// Contexts.
+package context
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+)
+
+// A Context carries a deadline, a cancelation signal, and other values across
+// API boundaries.
+//
+// Context's methods may be called by multiple goroutines simultaneously.
+type Context interface {
+	// Deadline returns the time when work done on behalf of this context
+	// should be canceled.  Deadline returns ok==false when no deadline is
+	// set.  Successive calls to Deadline return the same results.
+	Deadline() (deadline time.Time, ok bool)
+
+	// Done returns a channel that's closed when work done on behalf of this
+	// context should be canceled.  Done may return nil if this context can
+	// never be canceled.  Successive calls to Done return the same value.
+	//
+	// WithCancel arranges for Done to be closed when cancel is called;
+	// WithDeadline arranges for Done to be closed when the deadline
+	// expires; WithTimeout arranges for Done to be closed when the timeout
+	// elapses.
+	//
+	// Done is provided for use in select statements:
+	//
+	//  // Stream generates values with DoSomething and sends them to out
+	//  // until DoSomething returns an error or ctx.Done is closed.
+	//  func Stream(ctx context.Context, out <-chan Value) error {
+	//  	for {
+	//  		v, err := DoSomething(ctx)
+	//  		if err != nil {
+	//  			return err
+	//  		}
+	//  		select {
+	//  		case <-ctx.Done():
+	//  			return ctx.Err()
+	//  		case out <- v:
+	//  		}
+	//  	}
+	//  }
+	//
+	// See http://blog.golang.org/pipelines for more examples of how to use
+	// a Done channel for cancelation.
+	Done() <-chan struct{}
+
+	// Err returns a non-nil error value after Done is closed.  Err returns
+	// Canceled if the context was canceled or DeadlineExceeded if the
+	// context's deadline passed.  No other values for Err are defined.
+	// After Done is closed, successive calls to Err return the same value.
+	Err() error
+
+	// Value returns the value associated with this context for key, or nil
+	// if no value is associated with key.  Successive calls to Value with
+	// the same key returns the same result.
+	//
+	// Use context values only for request-scoped data that transits
+	// processes and API boundaries, not for passing optional parameters to
+	// functions.
+	//
+	// A key identifies a specific value in a Context.  Functions that wish
+	// to store values in Context typically allocate a key in a global
+	// variable then use that key as the argument to context.WithValue and
+	// Context.Value.  A key can be any type that supports equality;
+	// packages should define keys as an unexported type to avoid
+	// collisions.
+	//
+	// Packages that define a Context key should provide type-safe accessors
+	// for the values stores using that key:
+	//
+	// 	// Package user defines a User type that's stored in Contexts.
+	// 	package user
+	//
+	// 	import "golang.org/x/net/context"
+	//
+	// 	// User is the type of value stored in the Contexts.
+	// 	type User struct {...}
+	//
+	// 	// key is an unexported type for keys defined in this package.
+	// 	// This prevents collisions with keys defined in other packages.
+	// 	type key int
+	//
+	// 	// userKey is the key for user.User values in Contexts.  It is
+	// 	// unexported; clients use user.NewContext and user.FromContext
+	// 	// instead of using this key directly.
+	// 	var userKey key = 0
+	//
+	// 	// NewContext returns a new Context that carries value u.
+	// 	func NewContext(ctx context.Context, u *User) context.Context {
+	// 		return context.WithValue(ctx, userKey, u)
+	// 	}
+	//
+	// 	// FromContext returns the User value stored in ctx, if any.
+	// 	func FromContext(ctx context.Context) (*User, bool) {
+	// 		u, ok := ctx.Value(userKey).(*User)
+	// 		return u, ok
+	// 	}
+	Value(key interface{}) interface{}
+}
+
+// Canceled is the error returned by Context.Err when the context is canceled.
+var Canceled = errors.New("context canceled")
+
+// DeadlineExceeded is the error returned by Context.Err when the context's
+// deadline passes.
+var DeadlineExceeded = errors.New("context deadline exceeded")
+
+// An emptyCtx is never canceled, has no values, and has no deadline.  It is not
+// struct{}, since vars of this type must have distinct addresses.
+type emptyCtx int
+
+func (*emptyCtx) Deadline() (deadline time.Time, ok bool) {
+	return
+}
+
+func (*emptyCtx) Done() <-chan struct{} {
+	return nil
+}
+
+func (*emptyCtx) Err() error {
+	return nil
+}
+
+func (*emptyCtx) Value(key interface{}) interface{} {
+	return nil
+}
+
+func (e *emptyCtx) String() string {
+	switch e {
+	case background:
+		return "context.Background"
+	case todo:
+		return "context.TODO"
+	}
+	return "unknown empty Context"
+}
+
+var (
+	background = new(emptyCtx)
+	todo       = new(emptyCtx)
+)
+
+// Background returns a non-nil, empty Context. It is never canceled, has no
+// values, and has no deadline.  It is typically used by the main function,
+// initialization, and tests, and as the top-level Context for incoming
+// requests.
+func Background() Context {
+	return background
+}
+
+// TODO returns a non-nil, empty Context.  Code should use context.TODO when
+// it's unclear which Context to use or it's is not yet available (because the
+// surrounding function has not yet been extended to accept a Context
+// parameter).  TODO is recognized by static analysis tools that determine
+// whether Contexts are propagated correctly in a program.
+func TODO() Context {
+	return todo
+}
+
+// A CancelFunc tells an operation to abandon its work.
+// A CancelFunc does not wait for the work to stop.
+// After the first call, subsequent calls to a CancelFunc do nothing.
+type CancelFunc func()
+
+// WithCancel returns a copy of parent with a new Done channel. The returned
+// context's Done channel is closed when the returned cancel function is called
+// or when the parent context's Done channel is closed, whichever happens first.
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete.
+func WithCancel(parent Context) (ctx Context, cancel CancelFunc) {
+	c := newCancelCtx(parent)
+	propagateCancel(parent, &c)
+	return &c, func() { c.cancel(true, Canceled) }
+}
+
+// newCancelCtx returns an initialized cancelCtx.
+func newCancelCtx(parent Context) cancelCtx {
+	return cancelCtx{
+		Context: parent,
+		done:    make(chan struct{}),
+	}
+}
+
+// propagateCancel arranges for child to be canceled when parent is.
+func propagateCancel(parent Context, child canceler) {
+	if parent.Done() == nil {
+		return // parent is never canceled
+	}
+	if p, ok := parentCancelCtx(parent); ok {
+		p.mu.Lock()
+		if p.err != nil {
+			// parent has already been canceled
+			child.cancel(false, p.err)
+		} else {
+			if p.children == nil {
+				p.children = make(map[canceler]bool)
+			}
+			p.children[child] = true
+		}
+		p.mu.Unlock()
+	} else {
+		go func() {
+			select {
+			case <-parent.Done():
+				child.cancel(false, parent.Err())
+			case <-child.Done():
+			}
+		}()
+	}
+}
+
+// parentCancelCtx follows a chain of parent references until it finds a
+// *cancelCtx.  This function understands how each of the concrete types in this
+// package represents its parent.
+func parentCancelCtx(parent Context) (*cancelCtx, bool) {
+	for {
+		switch c := parent.(type) {
+		case *cancelCtx:
+			return c, true
+		case *timerCtx:
+			return &c.cancelCtx, true
+		case *valueCtx:
+			parent = c.Context
+		default:
+			return nil, false
+		}
+	}
+}
+
+// removeChild removes a context from its parent.
+func removeChild(parent Context, child canceler) {
+	p, ok := parentCancelCtx(parent)
+	if !ok {
+		return
+	}
+	p.mu.Lock()
+	if p.children != nil {
+		delete(p.children, child)
+	}
+	p.mu.Unlock()
+}
+
+// A canceler is a context type that can be canceled directly.  The
+// implementations are *cancelCtx and *timerCtx.
+type canceler interface {
+	cancel(removeFromParent bool, err error)
+	Done() <-chan struct{}
+}
+
+// A cancelCtx can be canceled.  When canceled, it also cancels any children
+// that implement canceler.
+type cancelCtx struct {
+	Context
+
+	done chan struct{} // closed by the first cancel call.
+
+	mu       sync.Mutex
+	children map[canceler]bool // set to nil by the first cancel call
+	err      error             // set to non-nil by the first cancel call
+}
+
+func (c *cancelCtx) Done() <-chan struct{} {
+	return c.done
+}
+
+func (c *cancelCtx) Err() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.err
+}
+
+func (c *cancelCtx) String() string {
+	return fmt.Sprintf("%v.WithCancel", c.Context)
+}
+
+// cancel closes c.done, cancels each of c's children, and, if
+// removeFromParent is true, removes c from its parent's children.
+func (c *cancelCtx) cancel(removeFromParent bool, err error) {
+	if err == nil {
+		panic("context: internal error: missing cancel error")
+	}
+	c.mu.Lock()
+	if c.err != nil {
+		c.mu.Unlock()
+		return // already canceled
+	}
+	c.err = err
+	close(c.done)
+	for child := range c.children {
+		// NOTE: acquiring the child's lock while holding parent's lock.
+		child.cancel(false, err)
+	}
+	c.children = nil
+	c.mu.Unlock()
+
+	if removeFromParent {
+		removeChild(c.Context, c)
+	}
+}
+
+// WithDeadline returns a copy of the parent context with the deadline adjusted
+// to be no later than d.  If the parent's deadline is already earlier than d,
+// WithDeadline(parent, d) is semantically equivalent to parent.  The returned
+// context's Done channel is closed when the deadline expires, when the returned
+// cancel function is called, or when the parent context's Done channel is
+// closed, whichever happens first.
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete.
+func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) {
+	if cur, ok := parent.Deadline(); ok && cur.Before(deadline) {
+		// The current deadline is already sooner than the new one.
+		return WithCancel(parent)
+	}
+	c := &timerCtx{
+		cancelCtx: newCancelCtx(parent),
+		deadline:  deadline,
+	}
+	propagateCancel(parent, c)
+	d := deadline.Sub(time.Now())
+	if d <= 0 {
+		c.cancel(true, DeadlineExceeded) // deadline has already passed
+		return c, func() { c.cancel(true, Canceled) }
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.err == nil {
+		c.timer = time.AfterFunc(d, func() {
+			c.cancel(true, DeadlineExceeded)
+		})
+	}
+	return c, func() { c.cancel(true, Canceled) }
+}
+
+// A timerCtx carries a timer and a deadline.  It embeds a cancelCtx to
+// implement Done and Err.  It implements cancel by stopping its timer then
+// delegating to cancelCtx.cancel.
+type timerCtx struct {
+	cancelCtx
+	timer *time.Timer // Under cancelCtx.mu.
+
+	deadline time.Time
+}
+
+func (c *timerCtx) Deadline() (deadline time.Time, ok bool) {
+	return c.deadline, true
+}
+
+func (c *timerCtx) String() string {
+	return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now()))
+}
+
+func (c *timerCtx) cancel(removeFromParent bool, err error) {
+	c.cancelCtx.cancel(false, err)
+	if removeFromParent {
+		// Remove this timerCtx from its parent cancelCtx's children.
+		removeChild(c.cancelCtx.Context, c)
+	}
+	c.mu.Lock()
+	if c.timer != nil {
+		c.timer.Stop()
+		c.timer = nil
+	}
+	c.mu.Unlock()
+}
+
+// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)).
+//
+// Canceling this context releases resources associated with it, so code should
+// call cancel as soon as the operations running in this Context complete:
+//
+// 	func slowOperationWithTimeout(ctx context.Context) (Result, error) {
+// 		ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
+// 		defer cancel()  // releases resources if slowOperation completes before timeout elapses
+// 		return slowOperation(ctx)
+// 	}
+func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) {
+	return WithDeadline(parent, time.Now().Add(timeout))
+}
+
+// WithValue returns a copy of parent in which the value associated with key is
+// val.
+//
+// Use context Values only for request-scoped data that transits processes and
+// APIs, not for passing optional parameters to functions.
+func WithValue(parent Context, key interface{}, val interface{}) Context {
+	return &valueCtx{parent, key, val}
+}
+
+// A valueCtx carries a key-value pair.  It implements Value for that key and
+// delegates all other calls to the embedded Context.
+type valueCtx struct {
+	Context
+	key, val interface{}
+}
+
+func (c *valueCtx) String() string {
+	return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val)
+}
+
+func (c *valueCtx) Value(key interface{}) interface{} {
+	if c.key == key {
+		return c.val
+	}
+	return c.Context.Value(key)
+}

Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/cancelreq.go

@@ -0,0 +1,18 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.5
+
+package ctxhttp
+
+import "net/http"
+
+func canceler(client *http.Client, req *http.Request) func() {
+	ch := make(chan struct{})
+	req.Cancel = ch
+
+	return func() {
+		close(ch)
+	}
+}

Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/cancelreq_go14.go

@@ -0,0 +1,23 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.5
+
+package ctxhttp
+
+import "net/http"
+
+type requestCanceler interface {
+	CancelRequest(*http.Request)
+}
+
+func canceler(client *http.Client, req *http.Request) func() {
+	rc, ok := client.Transport.(requestCanceler)
+	if !ok {
+		return func() {}
+	}
+	return func() {
+		rc.CancelRequest(req)
+	}
+}

Godeps/_workspace/src/golang.org/x/net/context/ctxhttp/ctxhttp.go

@@ -0,0 +1,79 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package ctxhttp provides helper functions for performing context-aware HTTP requests.
+package ctxhttp
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/golang.org/x/net/context"
+)
+
+// Do sends an HTTP request with the provided http.Client and returns an HTTP response.
+// If the client is nil, http.DefaultClient is used.
+// If the context is canceled or times out, ctx.Err() will be returned.
+func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
+	if client == nil {
+		client = http.DefaultClient
+	}
+
+	// Request cancelation changed in Go 1.5, see cancelreq.go and cancelreq_go14.go.
+	cancel := canceler(client, req)
+
+	type responseAndError struct {
+		resp *http.Response
+		err  error
+	}
+	result := make(chan responseAndError, 1)
+
+	go func() {
+		resp, err := client.Do(req)
+		result <- responseAndError{resp, err}
+	}()
+
+	select {
+	case <-ctx.Done():
+		cancel()
+		return nil, ctx.Err()
+	case r := <-result:
+		return r.resp, r.err
+	}
+}
+
+// Get issues a GET request via the Do function.
+func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return Do(ctx, client, req)
+}
+
+// Head issues a HEAD request via the Do function.
+func Head(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
+	req, err := http.NewRequest("HEAD", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return Do(ctx, client, req)
+}
+
+// Post issues a POST request via the Do function.
+func Post(ctx context.Context, client *http.Client, url string, bodyType string, body io.Reader) (*http.Response, error) {
+	req, err := http.NewRequest("POST", url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", bodyType)
+	return Do(ctx, client, req)
+}
+
+// PostForm issues a POST request via the Do function.
+func PostForm(ctx context.Context, client *http.Client, url string, data url.Values) (*http.Response, error) {
+	return Post(ctx, client, url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
+}

bdns/dns.go

@@ -12,7 +12,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/miekg/dns"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/golang.org/x/net/context"
 	"github.com/letsencrypt/boulder/metrics"
 )
 
@@ -114,19 +116,21 @@ var (
 	}
 )
 
-// DNSResolver defines methods used for DNS resolution
+// DNSResolver queries for DNS records
 type DNSResolver interface {
-	LookupTXT(string) ([]string, error)
+	LookupTXT(context.Context, string) ([]string, error)
-	LookupHost(string) ([]net.IP, error)
+	LookupHost(context.Context, string) ([]net.IP, error)
-	LookupCAA(string) ([]*dns.CAA, error)
+	LookupCAA(context.Context, string) ([]*dns.CAA, error)
-	LookupMX(string) ([]string, error)
+	LookupMX(context.Context, string) ([]string, error)
 }
 
 // DNSResolverImpl represents a client that talks to an external resolver
 type DNSResolverImpl struct {
-	DNSClient                *dns.Client
+	DNSClient                exchanger
 	Servers                  []string
 	allowRestrictedAddresses bool
+	maxTries                 int
+	clk                      clock.Clock
 	stats                    metrics.Scope
 	txtStats                 metrics.Scope
 	aStats                   metrics.Scope
@@ -136,9 +140,14 @@ type DNSResolverImpl struct {
 
 var _ DNSResolver = &DNSResolverImpl{}
 
+type exchanger interface {
+	Exchange(m *dns.Msg, a string) (*dns.Msg, time.Duration, error)
+}
+
 // NewDNSResolverImpl constructs a new DNS resolver object that utilizes the
 // provided list of DNS servers for resolution.
-func NewDNSResolverImpl(readTimeout time.Duration, servers []string, stats metrics.Scope) *DNSResolverImpl {
+func NewDNSResolverImpl(readTimeout time.Duration, servers []string, stats metrics.Scope, clk clock.Clock, maxTries int) *DNSResolverImpl {
+	// TODO(jmhodges): make constructor use an Option func pattern
 	dnsClient := new(dns.Client)
 
 	// Set timeout for underlying net.Conn
@@ -149,19 +158,21 @@ func NewDNSResolverImpl(readTimeout time.Duration, servers []string, stats metri
 		DNSClient:                dnsClient,
 		Servers:                  servers,
 		allowRestrictedAddresses: false,
-		stats:    stats,
+		maxTries:                 maxTries,
-		txtStats: stats.NewScope("TXT"),
+		clk:                      clk,
-		aStats:   stats.NewScope("A"),
+		stats:                    stats,
-		caaStats: stats.NewScope("CAA"),
+		txtStats:                 stats.NewScope("TXT"),
-		mxStats:  stats.NewScope("MX"),
+		aStats:                   stats.NewScope("A"),
+		caaStats:                 stats.NewScope("CAA"),
+		mxStats:                  stats.NewScope("MX"),
 	}
 }
 
 // NewTestDNSResolverImpl constructs a new DNS resolver object that utilizes the
 // provided list of DNS servers for resolution and will allow loopback addresses.
 // This constructor should *only* be called from tests (unit or integration).
-func NewTestDNSResolverImpl(readTimeout time.Duration, servers []string, stats metrics.Scope) *DNSResolverImpl {
+func NewTestDNSResolverImpl(readTimeout time.Duration, servers []string, stats metrics.Scope, clk clock.Clock, maxTries int) *DNSResolverImpl {
-	resolver := NewDNSResolverImpl(readTimeout, servers, stats)
+	resolver := NewDNSResolverImpl(readTimeout, servers, stats, clk, maxTries)
 	resolver.allowRestrictedAddresses = true
 	return resolver
 }
@@ -170,7 +181,7 @@ func NewTestDNSResolverImpl(readTimeout time.Duration, servers []string, stats m
 // out of the server list, returning the response, time, and error (if any).
 // This method sets the DNSSEC OK bit on the message to true before sending
 // it to the resolver in case validation isn't the resolvers default behaviour.
-func (dnsResolver *DNSResolverImpl) exchangeOne(hostname string, qtype uint16, msgStats metrics.Scope) (rsp *dns.Msg, err error) {
+func (dnsResolver *DNSResolverImpl) exchangeOne(ctx context.Context, hostname string, qtype uint16, msgStats metrics.Scope) (*dns.Msg, error) {
 	m := new(dns.Msg)
 	// Set question type
 	m.SetQuestion(dns.Fqdn(hostname), qtype)
@@ -178,8 +189,7 @@ func (dnsResolver *DNSResolverImpl) exchangeOne(hostname string, qtype uint16, m
 	m.SetEdns0(4096, true)
 
 	if len(dnsResolver.Servers) < 1 {
-		err = fmt.Errorf("Not configured with at least one DNS Server")
+		return nil, fmt.Errorf("Not configured with at least one DNS Server")
-		return
 	}
 
 	dnsResolver.stats.Inc("Rate", 1)
@@ -187,21 +197,58 @@ func (dnsResolver *DNSResolverImpl) exchangeOne(hostname string, qtype uint16, m
 	// Randomly pick a server
 	chosenServer := dnsResolver.Servers[rand.Intn(len(dnsResolver.Servers))]
 
-	msg, rtt, err := dnsResolver.DNSClient.Exchange(m, chosenServer)
+	client := dnsResolver.DNSClient
-	msgStats.TimingDuration("RTT", rtt)
+
-	if err == nil {
+	tries := 1
-		msgStats.Inc("Successes", 1)
+	start := dnsResolver.clk.Now()
-	} else {
+	msgStats.Inc("Calls", 1)
-		msgStats.Inc("Errors", 1)
+	defer msgStats.TimingDuration("Latency", dnsResolver.clk.Now().Sub(start))
+	for {
+		msgStats.Inc("Tries", 1)
+		ch := make(chan dnsResp, 1)
+
+		go func() {
+			rsp, rtt, err := client.Exchange(m, chosenServer)
+			msgStats.TimingDuration("SingleTryLatency", rtt)
+			ch <- dnsResp{m: rsp, err: err}
+		}()
+		select {
+		case <-ctx.Done():
+			msgStats.Inc("Cancels", 1)
+			msgStats.Inc("Errors", 1)
+			return nil, ctx.Err()
+		case r := <-ch:
+			if r.err != nil {
+				msgStats.Inc("Errors", 1)
+				operr, ok := r.err.(*net.OpError)
+				isRetryable := ok && operr.Temporary()
+				hasRetriesLeft := tries < dnsResolver.maxTries
+				if isRetryable && hasRetriesLeft {
+					tries++
+					continue
+				} else if isRetryable && !hasRetriesLeft {
+					msgStats.Inc("RanOutOfTries", 1)
+				}
+			} else {
+				msgStats.Inc("Successes", 1)
+			}
+			return r.m, r.err
+		}
 	}
-	return msg, err
 }
 
-// LookupTXT sends a DNS query to find all TXT records associated with
+type dnsResp struct {
-// the provided hostname.
+	m   *dns.Msg
-func (dnsResolver *DNSResolverImpl) LookupTXT(hostname string) ([]string, error) {
+	err error
+}
+
+// LookupTXT sends a DNS query to find all TXT records associated with the
+// provided hostname. It will retry requests in the case of temporary network
+// errors. It can return net package, context.Canceled, and
+// context.DeadlineExceeded errors.
+func (dnsResolver *DNSResolverImpl) LookupTXT(ctx context.Context, hostname string) ([]string, error) {
 	var txt []string
-	r, err := dnsResolver.exchangeOne(hostname, dns.TypeTXT, dnsResolver.txtStats)
+	r, err := dnsResolver.exchangeOne(ctx, hostname, dns.TypeTXT, dnsResolver.txtStats)
 	if err != nil {
 		return nil, err
 	}
@@ -230,13 +277,15 @@ func isPrivateV4(ip net.IP) bool {
 	return false
 }
 
-// LookupHost sends a DNS query to find all A records associated with the provided
+// LookupHost sends a DNS query to find all A records associated with the
-// hostname. This method assumes that the external resolver will chase CNAME/DNAME
+// provided hostname. This method assumes that the external resolver will chase
-// aliases and return relevant A records.
+// CNAME/DNAME aliases and return relevant A records.  It will retry requests in
-func (dnsResolver *DNSResolverImpl) LookupHost(hostname string) ([]net.IP, error) {
+// the case of temporary network errors. It can return net package,
+// context.Canceled, and context.DeadlineExceeded errors.
+func (dnsResolver *DNSResolverImpl) LookupHost(ctx context.Context, hostname string) ([]net.IP, error) {
 	var addrs []net.IP
 
-	r, err := dnsResolver.exchangeOne(hostname, dns.TypeA, dnsResolver.aStats)
+	r, err := dnsResolver.exchangeOne(ctx, hostname, dns.TypeA, dnsResolver.aStats)
 	if err != nil {
 		return addrs, err
 	}
@@ -256,11 +305,13 @@ func (dnsResolver *DNSResolverImpl) LookupHost(hostname string) ([]net.IP, error
 	return addrs, nil
 }
 
-// LookupCAA sends a DNS query to find all CAA records associated with
+// LookupCAA sends a DNS query to find all CAA records associated with the
-// the provided hostname. If the response code from the resolver is
+// provided hostname. If the response code from the resolver is SERVFAIL an
-// SERVFAIL an empty slice of CAA records is returned.
+// empty slice of CAA records is returned.  It will retry requests in the case
-func (dnsResolver *DNSResolverImpl) LookupCAA(hostname string) ([]*dns.CAA, error) {
+// of temporary network errors. It can return net package, context.Canceled, and
-	r, err := dnsResolver.exchangeOne(hostname, dns.TypeCAA, dnsResolver.caaStats)
+// context.DeadlineExceeded errors.
+func (dnsResolver *DNSResolverImpl) LookupCAA(ctx context.Context, hostname string) ([]*dns.CAA, error) {
+	r, err := dnsResolver.exchangeOne(ctx, hostname, dns.TypeCAA, dnsResolver.caaStats)
 	if err != nil {
 		return nil, err
 	}
@@ -282,10 +333,12 @@ func (dnsResolver *DNSResolverImpl) LookupCAA(hostname string) ([]*dns.CAA, erro
 	return CAAs, nil
 }
 
-// LookupMX sends a DNS query to find a MX record associated hostname and returns the
+// LookupMX sends a DNS query to find a MX record associated hostname and
-// record target.
+// returns the record target.  It will retry requests in the case of temporary
-func (dnsResolver *DNSResolverImpl) LookupMX(hostname string) ([]string, error) {
+// network errors. It can return net package, context.Canceled, and
-	r, err := dnsResolver.exchangeOne(hostname, dns.TypeMX, dnsResolver.mxStats)
+// context.DeadlineExceeded errors.
+func (dnsResolver *DNSResolverImpl) LookupMX(ctx context.Context, hostname string) ([]string, error) {
+	r, err := dnsResolver.exchangeOne(ctx, hostname, dns.TypeMX, dnsResolver.mxStats)
 	if err != nil {
 		return nil, err
 	}

bdns/dns_test.go

@@ -6,14 +6,19 @@
 package bdns
 
 import (
+	"errors"
 	"fmt"
 	"net"
 	"os"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/golang.org/x/net/context"
+
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/miekg/dns"
 	"github.com/letsencrypt/boulder/metrics"
 	"github.com/letsencrypt/boulder/test"
@@ -151,67 +156,67 @@ func newTestStats() metrics.Scope {
 var testStats = newTestStats()
 
 func TestDNSNoServers(t *testing.T) {
-	obj := NewTestDNSResolverImpl(time.Hour, []string{}, testStats)
+	obj := NewTestDNSResolverImpl(time.Hour, []string{}, testStats, clock.NewFake(), 1)
 
-	_, err := obj.LookupHost("letsencrypt.org")
+	_, err := obj.LookupHost(context.Background(), "letsencrypt.org")
 
 	test.AssertError(t, err, "No servers")
 }
 
 func TestDNSOneServer(t *testing.T) {
-	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats)
+	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats, clock.NewFake(), 1)
 
-	_, err := obj.LookupHost("letsencrypt.org")
+	_, err := obj.LookupHost(context.Background(), "letsencrypt.org")
 
 	test.AssertNotError(t, err, "No message")
 }
 
 func TestDNSDuplicateServers(t *testing.T) {
-	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr, dnsLoopbackAddr}, testStats)
+	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr, dnsLoopbackAddr}, testStats, clock.NewFake(), 1)
 
-	_, err := obj.LookupHost("letsencrypt.org")
+	_, err := obj.LookupHost(context.Background(), "letsencrypt.org")
 
 	test.AssertNotError(t, err, "No message")
 }
 
 func TestDNSLookupsNoServer(t *testing.T) {
-	obj := NewTestDNSResolverImpl(time.Second*10, []string{}, testStats)
+	obj := NewTestDNSResolverImpl(time.Second*10, []string{}, testStats, clock.NewFake(), 1)
 
-	_, err := obj.LookupTXT("letsencrypt.org")
+	_, err := obj.LookupTXT(context.Background(), "letsencrypt.org")
 	test.AssertError(t, err, "No servers")
 
-	_, err = obj.LookupHost("letsencrypt.org")
+	_, err = obj.LookupHost(context.Background(), "letsencrypt.org")
 	test.AssertError(t, err, "No servers")
 
-	_, err = obj.LookupCAA("letsencrypt.org")
+	_, err = obj.LookupCAA(context.Background(), "letsencrypt.org")
 	test.AssertError(t, err, "No servers")
 }
 
 func TestDNSServFail(t *testing.T) {
-	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats)
+	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats, clock.NewFake(), 1)
 	bad := "servfail.com"
 
-	_, err := obj.LookupTXT(bad)
+	_, err := obj.LookupTXT(context.Background(), bad)
 	test.AssertError(t, err, "LookupTXT didn't return an error")
 
-	_, err = obj.LookupHost(bad)
+	_, err = obj.LookupHost(context.Background(), bad)
 	test.AssertError(t, err, "LookupHost didn't return an error")
 
 	// CAA lookup ignores validation failures from the resolver for now
 	// and returns an empty list of CAA records.
-	emptyCaa, err := obj.LookupCAA(bad)
+	emptyCaa, err := obj.LookupCAA(context.Background(), bad)
 	test.Assert(t, len(emptyCaa) == 0, "Query returned non-empty list of CAA records")
 	test.AssertNotError(t, err, "LookupCAA returned an error")
 }
 
 func TestDNSLookupTXT(t *testing.T) {
-	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats)
+	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats, clock.NewFake(), 1)
 
-	a, err := obj.LookupTXT("letsencrypt.org")
+	a, err := obj.LookupTXT(context.Background(), "letsencrypt.org")
 	t.Logf("A: %v", a)
 	test.AssertNotError(t, err, "No message")
 
-	a, err = obj.LookupTXT("split-txt.letsencrypt.org")
+	a, err = obj.LookupTXT(context.Background(), "split-txt.letsencrypt.org")
 	t.Logf("A: %v ", a)
 	test.AssertNotError(t, err, "No message")
 	test.AssertEquals(t, len(a), 1)
@@ -219,47 +224,219 @@ func TestDNSLookupTXT(t *testing.T) {
 }
 
 func TestDNSLookupHost(t *testing.T) {
-	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats)
+	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats, clock.NewFake(), 1)
 
-	ip, err := obj.LookupHost("servfail.com")
+	ip, err := obj.LookupHost(context.Background(), "servfail.com")
 	t.Logf("servfail.com - IP: %s, Err: %s", ip, err)
 	test.AssertError(t, err, "Server failure")
 	test.Assert(t, len(ip) == 0, "Should not have IPs")
 
-	ip, err = obj.LookupHost("nonexistent.letsencrypt.org")
+	ip, err = obj.LookupHost(context.Background(), "nonexistent.letsencrypt.org")
 	t.Logf("nonexistent.letsencrypt.org - IP: %s, Err: %s", ip, err)
 	test.AssertNotError(t, err, "Not an error to not exist")
 	test.Assert(t, len(ip) == 0, "Should not have IPs")
 
 	// Single IPv4 address
-	ip, err = obj.LookupHost("cps.letsencrypt.org")
+	ip, err = obj.LookupHost(context.Background(), "cps.letsencrypt.org")
 	t.Logf("cps.letsencrypt.org - IP: %s, Err: %s", ip, err)
 	test.AssertNotError(t, err, "Not an error to exist")
 	test.Assert(t, len(ip) == 1, "Should have IP")
-	ip, err = obj.LookupHost("cps.letsencrypt.org")
+	ip, err = obj.LookupHost(context.Background(), "cps.letsencrypt.org")
 	t.Logf("cps.letsencrypt.org - IP: %s, Err: %s", ip, err)
 	test.AssertNotError(t, err, "Not an error to exist")
 	test.Assert(t, len(ip) == 1, "Should have IP")
 
 	// No IPv6
-	ip, err = obj.LookupHost("v6.letsencrypt.org")
+	ip, err = obj.LookupHost(context.Background(), "v6.letsencrypt.org")
 	t.Logf("v6.letsencrypt.org - IP: %s, Err: %s", ip, err)
 	test.AssertNotError(t, err, "Not an error to exist")
 	test.Assert(t, len(ip) == 0, "Should not have IPs")
 }
 
 func TestDNSLookupCAA(t *testing.T) {
-	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats)
+	obj := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats, clock.NewFake(), 1)
 
-	caas, err := obj.LookupCAA("bracewel.net")
+	caas, err := obj.LookupCAA(context.Background(), "bracewel.net")
 	test.AssertNotError(t, err, "CAA lookup failed")
 	test.Assert(t, len(caas) > 0, "Should have CAA records")
 
-	caas, err = obj.LookupCAA("nonexistent.letsencrypt.org")
+	caas, err = obj.LookupCAA(context.Background(), "nonexistent.letsencrypt.org")
 	test.AssertNotError(t, err, "CAA lookup failed")
 	test.Assert(t, len(caas) == 0, "Shouldn't have CAA records")
 
-	caas, err = obj.LookupCAA("cname.example.com")
+	caas, err = obj.LookupCAA(context.Background(), "cname.example.com")
 	test.AssertNotError(t, err, "CAA lookup failed")
 	test.Assert(t, len(caas) > 0, "Should follow CNAME to find CAA")
 }
+
+type testExchanger struct {
+	sync.Mutex
+	count int
+	errs  []error
+}
+
+var errTooManyRequests = errors.New("too many requests")
+
+func (te *testExchanger) Exchange(m *dns.Msg, a string) (*dns.Msg, time.Duration, error) {
+	te.Lock()
+	defer te.Unlock()
+	msg := &dns.Msg{
+		MsgHdr: dns.MsgHdr{Rcode: dns.RcodeSuccess},
+	}
+	if len(te.errs) <= te.count {
+		return nil, 0, errTooManyRequests
+	}
+	err := te.errs[te.count]
+	te.count++
+
+	return msg, 2 * time.Millisecond, err
+}
+
+func TestRetry(t *testing.T) {
+	isTempErr := &net.OpError{Op: "read", Err: tempError(true)}
+	nonTempErr := &net.OpError{Op: "read", Err: tempError(false)}
+	type testCase struct {
+		maxTries int
+		expected int
+		te       *testExchanger
+	}
+	tests := []*testCase{
+		// The success on first try case
+		{
+			maxTries: 3,
+			expected: 1,
+			te: &testExchanger{
+				errs: []error{nil},
+			},
+		},
+		// Immediate non-OpError, error returns immediately
+		{
+			maxTries: 3,
+			expected: 1,
+			te: &testExchanger{
+				errs: []error{errors.New("nope")},
+			},
+		},
+		// Temporary err, then non-OpError stops at two tries
+		{
+			maxTries: 3,
+			expected: 2,
+			te: &testExchanger{
+				errs: []error{isTempErr, errors.New("nope")},
+			},
+		},
+		// Temporary error given always
+		{
+			maxTries: 3,
+			expected: 3,
+			te: &testExchanger{
+				errs: []error{
+					isTempErr,
+					isTempErr,
+					isTempErr,
+				},
+			},
+		},
+		// Even with maxTries at 0, we should still let a single request go
+		// through
+		{
+			maxTries: 0,
+			expected: 1,
+			te: &testExchanger{
+				errs: []error{nil},
+			},
+		},
+		// Temporary error given just once causes two tries
+		{
+			maxTries: 3,
+			expected: 2,
+			te: &testExchanger{
+				errs: []error{
+					isTempErr,
+					nil,
+				},
+			},
+		},
+		// Temporary error given twice causes three tries
+		{
+			maxTries: 3,
+			expected: 3,
+			te: &testExchanger{
+				errs: []error{
+					isTempErr,
+					isTempErr,
+					nil,
+				},
+			},
+		},
+		// Temporary error given thrice causes three tries and fails
+		{
+			maxTries: 3,
+			expected: 3,
+			te: &testExchanger{
+				errs: []error{
+					isTempErr,
+					isTempErr,
+					isTempErr,
+				},
+			},
+		},
+		// temporary then non-Temporary error causes two retries
+		{
+			maxTries: 3,
+			expected: 2,
+			te: &testExchanger{
+				errs: []error{
+					isTempErr,
+					nonTempErr,
+				},
+			},
+		},
+	}
+
+	for i, tc := range tests {
+		dr := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats, clock.NewFake(), tc.maxTries)
+
+		dr.DNSClient = tc.te
+		_, err := dr.LookupTXT(context.Background(), "example.com")
+		if err == errTooManyRequests {
+			t.Errorf("#%d, sent more requests than the test case handles", i)
+		}
+		expectedErr := tc.te.errs[tc.expected-1]
+		if err != expectedErr {
+			t.Errorf("#%d, error, expected %v, got %v", i, expectedErr, err)
+		}
+		if tc.expected != tc.te.count {
+			t.Errorf("#%d, count, expected %d, got %d", i, tc.expected, tc.te.count)
+		}
+	}
+
+	dr := NewTestDNSResolverImpl(time.Second*10, []string{dnsLoopbackAddr}, testStats, clock.NewFake(), 3)
+	dr.DNSClient = &testExchanger{errs: []error{isTempErr, isTempErr, nil}}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	_, err := dr.LookupTXT(ctx, "example.com")
+	if err != context.Canceled {
+		t.Errorf("expected %s, got %s", context.Canceled, err)
+	}
+
+	dr.DNSClient = &testExchanger{errs: []error{isTempErr, isTempErr, nil}}
+	ctx, _ = context.WithTimeout(context.Background(), -10*time.Hour)
+	_, err = dr.LookupTXT(ctx, "example.com")
+	if err != context.DeadlineExceeded {
+		t.Errorf("expected %s, got %s", context.DeadlineExceeded, err)
+	}
+
+	dr.DNSClient = &testExchanger{errs: []error{isTempErr, isTempErr, nil}}
+	ctx, deadlineCancel := context.WithTimeout(context.Background(), -10*time.Hour)
+	deadlineCancel()
+	_, err = dr.LookupTXT(ctx, "example.com")
+	if err != context.DeadlineExceeded {
+		t.Errorf("expected %s, got %s", context.DeadlineExceeded, err)
+	}
+}
+
+type tempError bool
+
+func (t tempError) Temporary() bool { return bool(t) }
+func (t tempError) Error() string   { return fmt.Sprintf("Temporary: %t", t) }

cmd/boulder-ra/main.go

@@ -64,10 +64,14 @@ func main() {
 		raDNSTimeout, err := time.ParseDuration(c.Common.DNSTimeout)
 		cmd.FailOnError(err, "Couldn't parse RA DNS timeout")
 		scoped := metrics.NewStatsdScope(stats, "RA", "DNS")
+		dnsTries := c.RA.DNSTries
+		if dnsTries < 1 {
+			dnsTries = 1
+		}
 		if !c.Common.DNSAllowLoopbackAddresses {
-			rai.DNSResolver = bdns.NewDNSResolverImpl(raDNSTimeout, []string{c.Common.DNSResolver}, scoped)
+			rai.DNSResolver = bdns.NewDNSResolverImpl(raDNSTimeout, []string{c.Common.DNSResolver}, scoped, clock.Default(), dnsTries)
 		} else {
-			rai.DNSResolver = bdns.NewTestDNSResolverImpl(raDNSTimeout, []string{c.Common.DNSResolver}, scoped)
+			rai.DNSResolver = bdns.NewTestDNSResolverImpl(raDNSTimeout, []string{c.Common.DNSResolver}, scoped, clock.Default(), dnsTries)
 		}
 
 		rai.VA = vac

cmd/boulder-va/main.go

@@ -42,15 +42,20 @@ func main() {
 		if c.VA.PortConfig.TLSPort != 0 {
 			pc.TLSPort = c.VA.PortConfig.TLSPort
 		}
+		clk := clock.Default()
 		sbc := newGoogleSafeBrowsing(c.VA.GoogleSafeBrowsing)
-		vai := va.NewValidationAuthorityImpl(pc, sbc, stats, clock.Default())
+		vai := va.NewValidationAuthorityImpl(pc, sbc, stats, clk)
 		dnsTimeout, err := time.ParseDuration(c.Common.DNSTimeout)
 		cmd.FailOnError(err, "Couldn't parse DNS timeout")
 		scoped := metrics.NewStatsdScope(stats, "VA", "DNS")
+		dnsTries := c.VA.DNSTries
+		if dnsTries < 1 {
+			dnsTries = 1
+		}
 		if !c.Common.DNSAllowLoopbackAddresses {
-			vai.DNSResolver = bdns.NewDNSResolverImpl(dnsTimeout, []string{c.Common.DNSResolver}, scoped)
+			vai.DNSResolver = bdns.NewDNSResolverImpl(dnsTimeout, []string{c.Common.DNSResolver}, scoped, clk, dnsTries)
 		} else {
-			vai.DNSResolver = bdns.NewTestDNSResolverImpl(dnsTimeout, []string{c.Common.DNSResolver}, scoped)
+			vai.DNSResolver = bdns.NewTestDNSResolverImpl(dnsTimeout, []string{c.Common.DNSResolver}, scoped, clk, dnsTries)
 		}
 		vai.UserAgent = c.VA.UserAgent
 		vai.IssuerDomain = c.VA.IssuerDomain

cmd/config.go

@@ -62,6 +62,11 @@ type Config struct {
 
 		// UseIsSafeDomain determines whether to call VA.IsSafeDomain
 		UseIsSafeDomain bool // TODO(jmhodges): remove after va IsSafeDomain deploy
+
+		// The number of times to try a DNS query (that has a temporary error)
+		// before giving up. May be short-circuited by deadlines. A zero value
+		// will be turned into 1.
+		DNSTries int
 	}
 
 	SA struct {
@@ -83,6 +88,11 @@ type Config struct {
 		MaxConcurrentRPCServerRequests int64
 
 		GoogleSafeBrowsing *GoogleSafeBrowsingConfig
+
+		// The number of times to try a DNS query (that has a temporary error)
+		// before giving up. May be short-circuited by deadlines. A zero value
+		// will be turned into 1.
+		DNSTries int
 	}
 
 	SQL struct {

mocks/mocks.go

@@ -24,6 +24,7 @@ import (
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/go-jose"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/miekg/dns"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/golang.org/x/net/context"
 
 	"github.com/letsencrypt/boulder/core"
 )
@@ -33,7 +34,7 @@ type DNSResolver struct {
 }
 
 // LookupTXT is a mock
-func (mock *DNSResolver) LookupTXT(hostname string) ([]string, error) {
+func (mock *DNSResolver) LookupTXT(ctx context.Context, hostname string) ([]string, error) {
 	if hostname == "_acme-challenge.servfail.com" {
 		return nil, fmt.Errorf("SERVFAIL")
 	}
@@ -66,7 +67,7 @@ func (t timeoutError) Timeout() bool {
 //
 // Note: see comments on LookupMX regarding email.only
 //
-func (mock *DNSResolver) LookupHost(hostname string) ([]net.IP, error) {
+func (mock *DNSResolver) LookupHost(ctx context.Context, hostname string) ([]net.IP, error) {
 	if hostname == "always.invalid" ||
 		hostname == "invalid.invalid" ||
 		hostname == "email.only" {
@@ -85,7 +86,7 @@ func (mock *DNSResolver) LookupHost(hostname string) ([]net.IP, error) {
 }
 
 // LookupCAA is a mock
-func (mock *DNSResolver) LookupCAA(domain string) ([]*dns.CAA, error) {
+func (mock *DNSResolver) LookupCAA(ctx context.Context, domain string) ([]*dns.CAA, error) {
 	var results []*dns.CAA
 	var record dns.CAA
 	switch strings.TrimRight(domain, ".") {
@@ -121,7 +122,7 @@ func (mock *DNSResolver) LookupCAA(domain string) ([]*dns.CAA, error) {
 // all domains except for special cases, so MX-only domains must be
 // handled in both LookupHost and LookupMX.
 //
-func (mock *DNSResolver) LookupMX(domain string) ([]string, error) {
+func (mock *DNSResolver) LookupMX(ctx context.Context, domain string) ([]string, error) {
 	switch strings.TrimRight(domain, ".") {
 	case "letsencrypt.org":
 		fallthrough

ra/registration-authority.go

@@ -20,6 +20,7 @@ import (
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/net/publicsuffix"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/golang.org/x/net/context"
 	"github.com/letsencrypt/boulder/probs"
 
 	"github.com/letsencrypt/boulder/bdns"
@@ -84,7 +85,7 @@ const (
 	emptyDNSResponseDetail = "empty DNS response"
 )
 
-func validateEmail(address string, resolver bdns.DNSResolver) (prob *probs.ProblemDetails) {
+func validateEmail(ctx context.Context, address string, resolver bdns.DNSResolver) (prob *probs.ProblemDetails) {
 	_, err := mail.ParseAddress(address)
 	if err != nil {
 		return &probs.ProblemDetails{
@@ -96,10 +97,10 @@ func validateEmail(address string, resolver bdns.DNSResolver) (prob *probs.Probl
 	domain := strings.ToLower(splitEmail[len(splitEmail)-1])
 	var resultMX []string
 	var resultA []net.IP
-	resultMX, err = resolver.LookupMX(domain)
+	resultMX, err = resolver.LookupMX(ctx, domain)
 	recQ := "MX"
 	if err == nil && len(resultMX) == 0 {
-		resultA, err = resolver.LookupHost(domain)
+		resultA, err = resolver.LookupHost(ctx, domain)
 		recQ = "A"
 		if err == nil && len(resultA) == 0 {
 			return &probs.ProblemDetails{
@@ -209,7 +210,8 @@ func (ra *RegistrationAuthorityImpl) NewRegistration(init core.Registration) (re
 	// MergeUpdate. But we need to fill it in for new registrations.
 	reg.InitialIP = init.InitialIP
 
-	err = ra.validateContacts(reg.Contact)
+	// TODO(#1292): add a proper deadline here
+	err = ra.validateContacts(context.TODO(), reg.Contact)
 	if err != nil {
 		return
 	}
@@ -226,7 +228,7 @@ func (ra *RegistrationAuthorityImpl) NewRegistration(init core.Registration) (re
 	return
 }
 
-func (ra *RegistrationAuthorityImpl) validateContacts(contacts []*core.AcmeURL) (err error) {
+func (ra *RegistrationAuthorityImpl) validateContacts(ctx context.Context, contacts []*core.AcmeURL) (err error) {
 	if ra.maxContactsPerReg > 0 && len(contacts) > ra.maxContactsPerReg {
 		return core.MalformedRequestError(fmt.Sprintf("Too many contacts provided: %d > %d",
 			len(contacts), ra.maxContactsPerReg))
@@ -242,7 +244,7 @@ func (ra *RegistrationAuthorityImpl) validateContacts(contacts []*core.AcmeURL)
 		case "mailto":
 			start := ra.clk.Now()
 			ra.stats.Inc("RA.ValidateEmail.Calls", 1, 1.0)
-			problem := validateEmail(contact.Opaque, ra.DNSResolver)
+			problem := validateEmail(ctx, contact.Opaque, ra.DNSResolver)
 			ra.stats.TimingDuration("RA.ValidateEmail.Latency", ra.clk.Now().Sub(start), 1.0)
 			if problem != nil {
 				ra.stats.Inc("RA.ValidateEmail.Errors", 1, 1.0)
@@ -638,7 +640,8 @@ func (ra *RegistrationAuthorityImpl) checkLimits(names []string, regID int64) er
 func (ra *RegistrationAuthorityImpl) UpdateRegistration(base core.Registration, update core.Registration) (reg core.Registration, err error) {
 	base.MergeUpdate(update)
 
-	err = ra.validateContacts(base.Contact)
+	// TODO(#1292): add a proper deadline here
+	err = ra.validateContacts(context.TODO(), base.Contact)
 	if err != nil {
 		return
 	}

ra/registration-authority_test.go

@@ -22,6 +22,7 @@ import (
 	cfsslConfig "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/config"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	jose "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/go-jose"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/golang.org/x/net/context"
 	"github.com/letsencrypt/boulder/ca"
 	"github.com/letsencrypt/boulder/cmd"
 	"github.com/letsencrypt/boulder/core"
@@ -288,25 +289,25 @@ func TestValidateContacts(t *testing.T) {
 	validEmail, _ := core.ParseAcmeURL("mailto:admin@email.com")
 	malformedEmail, _ := core.ParseAcmeURL("mailto:admin.com")
 
-	err := ra.validateContacts([]*core.AcmeURL{})
+	err := ra.validateContacts(context.Background(), []*core.AcmeURL{})
 	test.AssertNotError(t, err, "No Contacts")
 
-	err = ra.validateContacts([]*core.AcmeURL{tel, validEmail})
+	err = ra.validateContacts(context.Background(), []*core.AcmeURL{tel, validEmail})
 	test.AssertError(t, err, "Too Many Contacts")
 
-	err = ra.validateContacts([]*core.AcmeURL{tel})
+	err = ra.validateContacts(context.Background(), []*core.AcmeURL{tel})
 	test.AssertNotError(t, err, "Simple Telephone")
 
-	err = ra.validateContacts([]*core.AcmeURL{validEmail})
+	err = ra.validateContacts(context.Background(), []*core.AcmeURL{validEmail})
 	test.AssertNotError(t, err, "Valid Email")
 
-	err = ra.validateContacts([]*core.AcmeURL{malformedEmail})
+	err = ra.validateContacts(context.Background(), []*core.AcmeURL{malformedEmail})
 	test.AssertError(t, err, "Malformed Email")
 
-	err = ra.validateContacts([]*core.AcmeURL{ansible})
+	err = ra.validateContacts(context.Background(), []*core.AcmeURL{ansible})
 	test.AssertError(t, err, "Unknown scheme")
 
-	err = ra.validateContacts([]*core.AcmeURL{nil})
+	err = ra.validateContacts(context.Background(), []*core.AcmeURL{nil})
 	test.AssertError(t, err, "Nil AcmeURL")
 }
 
@@ -324,8 +325,9 @@ func TestValidateEmail(t *testing.T) {
 		"a@email.com",
 		"b@email.only",
 	}
+
 	for _, tc := range testFailures {
-		problem := validateEmail(tc.input, &mocks.DNSResolver{})
+		problem := validateEmail(context.Background(), tc.input, &mocks.DNSResolver{})
 		if problem.Type != probs.InvalidEmailProblem {
 			t.Errorf("validateEmail(%q): got problem type %#v, expected %#v", tc.input, problem.Type, probs.InvalidEmailProblem)
 		}
@@ -336,7 +338,7 @@ func TestValidateEmail(t *testing.T) {
 	}
 
 	for _, addr := range testSuccesses {
-		if prob := validateEmail(addr, &mocks.DNSResolver{}); prob != nil {
+		if prob := validateEmail(context.Background(), addr, &mocks.DNSResolver{}); prob != nil {
 			t.Errorf("validateEmail(%q): expected success, but it failed: %s",
 				addr, prob)
 		}

test/boulder-config.json

@@ -123,6 +123,7 @@
     "rateLimitPoliciesFilename": "test/rate-limit-policies.yml",
     "maxConcurrentRPCServerRequests": 16,
     "maxContactsPerRegistration": 100,
+    "dnsTries": 3,
     "debugAddr": "localhost:8002",
     "amqp": {
       "serverURLFile": "test/secrets/amqp_url",
@@ -165,6 +166,7 @@
       "tlsPort": 5001
     },
     "maxConcurrentRPCServerRequests": 16,
+    "dnsTries": 3,
     "amqp": {
       "serverURLFile": "test/secrets/amqp_url",
       "insecure": true,

va/validation-authority.go

@@ -24,6 +24,7 @@ import (
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/net/publicsuffix"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/miekg/dns"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/golang.org/x/net/context"
 	"github.com/letsencrypt/boulder/probs"
 
 	"github.com/letsencrypt/boulder/bdns"
@@ -89,8 +90,8 @@ type verificationRequestEvent struct {
 // This is the same choice made by the Go internal resolution library used by
 // net/http, except we only send A queries and accept IPv4 addresses.
 // TODO(#593): Add IPv6 support
-func (va ValidationAuthorityImpl) getAddr(hostname string) (net.IP, []net.IP, *probs.ProblemDetails) {
+func (va ValidationAuthorityImpl) getAddr(ctx context.Context, hostname string) (net.IP, []net.IP, *probs.ProblemDetails) {
-	addrs, err := va.DNSResolver.LookupHost(hostname)
+	addrs, err := va.DNSResolver.LookupHost(ctx, hostname)
 	if err != nil {
 		va.log.Debug(fmt.Sprintf("%s DNS failure: %s", hostname, err))
 		problem := bdns.ProblemDetailsFromDNSError("A", hostname, err)
@@ -120,7 +121,7 @@ func (d *dialer) Dial(_, _ string) (net.Conn, error) {
 
 // resolveAndConstructDialer gets the prefered address using va.getAddr and returns
 // the chosen address and dialer for that address and correct port.
-func (va *ValidationAuthorityImpl) resolveAndConstructDialer(name string, port int) (dialer, *probs.ProblemDetails) {
+func (va *ValidationAuthorityImpl) resolveAndConstructDialer(ctx context.Context, name string, port int) (dialer, *probs.ProblemDetails) {
 	d := dialer{
 		record: core.ValidationRecord{
 			Hostname: name,
@@ -128,7 +129,7 @@ func (va *ValidationAuthorityImpl) resolveAndConstructDialer(name string, port i
 		},
 	}
 
-	addr, allAddrs, err := va.getAddr(name)
+	addr, allAddrs, err := va.getAddr(ctx, name)
 	if err != nil {
 		return d, err
 	}
@@ -139,7 +140,7 @@ func (va *ValidationAuthorityImpl) resolveAndConstructDialer(name string, port i
 
 // Validation methods
 
-func (va *ValidationAuthorityImpl) fetchHTTP(identifier core.AcmeIdentifier, path string, useTLS bool, input core.Challenge) ([]byte, []core.ValidationRecord, *probs.ProblemDetails) {
+func (va *ValidationAuthorityImpl) fetchHTTP(ctx context.Context, identifier core.AcmeIdentifier, path string, useTLS bool, input core.Challenge) ([]byte, []core.ValidationRecord, *probs.ProblemDetails) {
 	challenge := input
 
 	host := identifier.Value
@@ -177,7 +178,7 @@ func (va *ValidationAuthorityImpl) fetchHTTP(identifier core.AcmeIdentifier, pat
 		httpRequest.Header["User-Agent"] = []string{va.UserAgent}
 	}
 
-	dialer, prob := va.resolveAndConstructDialer(host, port)
+	dialer, prob := va.resolveAndConstructDialer(ctx, host, port)
 	dialer.record.URL = url.String()
 	validationRecords := []core.ValidationRecord{dialer.record}
 	if prob != nil {
@@ -236,7 +237,7 @@ func (va *ValidationAuthorityImpl) fetchHTTP(identifier core.AcmeIdentifier, pat
 			reqPort = 80
 		}
 
-		dialer, err := va.resolveAndConstructDialer(reqHost, reqPort)
+		dialer, err := va.resolveAndConstructDialer(ctx, reqHost, reqPort)
 		dialer.record.URL = req.URL.String()
 		validationRecords = append(validationRecords, dialer.record)
 		if err != nil {
@@ -279,8 +280,8 @@ func (va *ValidationAuthorityImpl) fetchHTTP(identifier core.AcmeIdentifier, pat
 	return body, validationRecords, nil
 }
 
-func (va *ValidationAuthorityImpl) validateTLSWithZName(identifier core.AcmeIdentifier, challenge core.Challenge, zName string) ([]core.ValidationRecord, *probs.ProblemDetails) {
+func (va *ValidationAuthorityImpl) validateTLSWithZName(ctx context.Context, identifier core.AcmeIdentifier, challenge core.Challenge, zName string) ([]core.ValidationRecord, *probs.ProblemDetails) {
-	addr, allAddrs, problem := va.getAddr(identifier.Value)
+	addr, allAddrs, problem := va.getAddr(ctx, identifier.Value)
 	validationRecords := []core.ValidationRecord{
 		core.ValidationRecord{
 			Hostname:          identifier.Value,
@@ -332,7 +333,7 @@ func (va *ValidationAuthorityImpl) validateTLSWithZName(identifier core.AcmeIden
 	}
 }
 
-func (va *ValidationAuthorityImpl) validateHTTP01(identifier core.AcmeIdentifier, challenge core.Challenge) ([]core.ValidationRecord, *probs.ProblemDetails) {
+func (va *ValidationAuthorityImpl) validateHTTP01(ctx context.Context, identifier core.AcmeIdentifier, challenge core.Challenge) ([]core.ValidationRecord, *probs.ProblemDetails) {
 	if identifier.Type != core.IdentifierDNS {
 		va.log.Debug(fmt.Sprintf("%s [%s] Identifier failure", challenge.Type, identifier))
 		return nil, &probs.ProblemDetails{
@@ -343,7 +344,7 @@ func (va *ValidationAuthorityImpl) validateHTTP01(identifier core.AcmeIdentifier
 
 	// Perform the fetch
 	path := fmt.Sprintf(".well-known/acme-challenge/%s", challenge.Token)
-	body, validationRecords, err := va.fetchHTTP(identifier, path, false, challenge)
+	body, validationRecords, err := va.fetchHTTP(ctx, identifier, path, false, challenge)
 	if err != nil {
 		return validationRecords, err
 	}
@@ -374,7 +375,7 @@ func (va *ValidationAuthorityImpl) validateHTTP01(identifier core.AcmeIdentifier
 	return validationRecords, nil
 }
 
-func (va *ValidationAuthorityImpl) validateTLSSNI01(identifier core.AcmeIdentifier, challenge core.Challenge) ([]core.ValidationRecord, *probs.ProblemDetails) {
+func (va *ValidationAuthorityImpl) validateTLSSNI01(ctx context.Context, identifier core.AcmeIdentifier, challenge core.Challenge) ([]core.ValidationRecord, *probs.ProblemDetails) {
 	if identifier.Type != "dns" {
 		va.log.Debug(fmt.Sprintf("TLS-SNI [%s] Identifier failure", identifier))
 		return nil, &probs.ProblemDetails{
@@ -389,7 +390,7 @@ func (va *ValidationAuthorityImpl) validateTLSSNI01(identifier core.AcmeIdentifi
 	Z := hex.EncodeToString(h.Sum(nil))
 	ZName := fmt.Sprintf("%s.%s.%s", Z[:32], Z[32:], core.TLSSNISuffix)
 
-	return va.validateTLSWithZName(identifier, challenge, ZName)
+	return va.validateTLSWithZName(ctx, identifier, challenge, ZName)
 }
 
 // parseHTTPConnError returns the ACME ProblemType corresponding to an error
@@ -414,7 +415,7 @@ func parseHTTPConnError(err error) probs.ProblemType {
 	return probs.ConnectionProblem
 }
 
-func (va *ValidationAuthorityImpl) validateDNS01(identifier core.AcmeIdentifier, challenge core.Challenge) ([]core.ValidationRecord, *probs.ProblemDetails) {
+func (va *ValidationAuthorityImpl) validateDNS01(ctx context.Context, identifier core.AcmeIdentifier, challenge core.Challenge) ([]core.ValidationRecord, *probs.ProblemDetails) {
 	if identifier.Type != core.IdentifierDNS {
 		va.log.Debug(fmt.Sprintf("DNS [%s] Identifier failure", identifier))
 		return nil, &probs.ProblemDetails{
@@ -430,7 +431,7 @@ func (va *ValidationAuthorityImpl) validateDNS01(identifier core.AcmeIdentifier,
 
 	// Look for the required record in the DNS
 	challengeSubdomain := fmt.Sprintf("%s.%s", core.DNSPrefix, identifier.Value)
-	txts, err := va.DNSResolver.LookupTXT(challengeSubdomain)
+	txts, err := va.DNSResolver.LookupTXT(ctx, challengeSubdomain)
 
 	if err != nil {
 		va.log.Debug(fmt.Sprintf("%s [%s] DNS failure: %s", challenge.Type, identifier, err))
@@ -451,9 +452,9 @@ func (va *ValidationAuthorityImpl) validateDNS01(identifier core.AcmeIdentifier,
 	}
 }
 
-func (va *ValidationAuthorityImpl) checkCAA(identifier core.AcmeIdentifier, regID int64) *probs.ProblemDetails {
+func (va *ValidationAuthorityImpl) checkCAA(ctx context.Context, identifier core.AcmeIdentifier, regID int64) *probs.ProblemDetails {
 	// Check CAA records for the requested identifier
-	present, valid, err := va.CheckCAARecords(identifier)
+	present, valid, err := va.checkCAARecords(ctx, identifier)
 	if err != nil {
 		va.log.Warning(fmt.Sprintf("Problem checking CAA: %s", err))
 		return bdns.ProblemDetailsFromDNSError("CAA", identifier.Value, err)
@@ -471,7 +472,7 @@ func (va *ValidationAuthorityImpl) checkCAA(identifier core.AcmeIdentifier, regI
 
 // Overall validation process
 
-func (va *ValidationAuthorityImpl) validate(authz core.Authorization, challengeIndex int) {
+func (va *ValidationAuthorityImpl) validate(ctx context.Context, authz core.Authorization, challengeIndex int) {
 	logEvent := verificationRequestEvent{
 		ID:          authz.ID,
 		Requester:   authz.RegistrationID,
@@ -479,7 +480,7 @@ func (va *ValidationAuthorityImpl) validate(authz core.Authorization, challengeI
 	}
 	challenge := &authz.Challenges[challengeIndex]
 	vStart := va.clk.Now()
-	validationRecords, prob := va.validateChallengeAndCAA(authz.Identifier, *challenge, authz.RegistrationID)
+	validationRecords, prob := va.validateChallengeAndCAA(ctx, authz.Identifier, *challenge, authz.RegistrationID)
 	va.stats.TimingDuration(fmt.Sprintf("VA.Validations.%s.%s", challenge.Type, challenge.Status), time.Since(vStart), 1.0)
 
 	challenge.ValidationRecord = validationRecords
@@ -505,13 +506,14 @@ func (va *ValidationAuthorityImpl) validate(authz core.Authorization, challengeI
 	va.RA.OnValidationUpdate(authz)
 }
 
-func (va *ValidationAuthorityImpl) validateChallengeAndCAA(identifier core.AcmeIdentifier, challenge core.Challenge, regID int64) ([]core.ValidationRecord, *probs.ProblemDetails) {
+func (va *ValidationAuthorityImpl) validateChallengeAndCAA(ctx context.Context, identifier core.AcmeIdentifier, challenge core.Challenge, regID int64) ([]core.ValidationRecord, *probs.ProblemDetails) {
 	ch := make(chan *probs.ProblemDetails, 1)
 	go func() {
-		ch <- va.checkCAA(identifier, regID)
+		ch <- va.checkCAA(ctx, identifier, regID)
 	}()
 
-	validationRecords, err := va.validateChallenge(identifier, challenge)
+	// TODO(#1292): send into another goroutine
+	validationRecords, err := va.validateChallenge(ctx, identifier, challenge)
 	if err != nil {
 		return validationRecords, err
 	}
@@ -523,7 +525,7 @@ func (va *ValidationAuthorityImpl) validateChallengeAndCAA(identifier core.AcmeI
 	return validationRecords, nil
 }
 
-func (va *ValidationAuthorityImpl) validateChallenge(identifier core.AcmeIdentifier, challenge core.Challenge) ([]core.ValidationRecord, *probs.ProblemDetails) {
+func (va *ValidationAuthorityImpl) validateChallenge(ctx context.Context, identifier core.AcmeIdentifier, challenge core.Challenge) ([]core.ValidationRecord, *probs.ProblemDetails) {
 	if !challenge.IsSane(true) {
 		return nil, &probs.ProblemDetails{
 			Type:   probs.MalformedProblem,
@@ -532,11 +534,11 @@ func (va *ValidationAuthorityImpl) validateChallenge(identifier core.AcmeIdentif
 	}
 	switch challenge.Type {
 	case core.ChallengeTypeHTTP01:
-		return va.validateHTTP01(identifier, challenge)
+		return va.validateHTTP01(ctx, identifier, challenge)
 	case core.ChallengeTypeTLSSNI01:
-		return va.validateTLSSNI01(identifier, challenge)
+		return va.validateTLSSNI01(ctx, identifier, challenge)
 	case core.ChallengeTypeDNS01:
-		return va.validateDNS01(identifier, challenge)
+		return va.validateDNS01(ctx, identifier, challenge)
 	}
 	return nil, &probs.ProblemDetails{
 		Type:   probs.MalformedProblem,
@@ -546,7 +548,8 @@ func (va *ValidationAuthorityImpl) validateChallenge(identifier core.AcmeIdentif
 
 // UpdateValidations runs the validate() method asynchronously using goroutines.
 func (va *ValidationAuthorityImpl) UpdateValidations(authz core.Authorization, challengeIndex int) error {
-	go va.validate(authz, challengeIndex)
+	// TODO(#1292): add a proper deadline here
+	go va.validate(context.TODO(), authz, challengeIndex)
 	return nil
 }
 
@@ -593,7 +596,7 @@ func newCAASet(CAAs []*dns.CAA) *CAASet {
 	return &filtered
 }
 
-func (va *ValidationAuthorityImpl) getCAASet(hostname string) (*CAASet, error) {
+func (va *ValidationAuthorityImpl) getCAASet(ctx context.Context, hostname string) (*CAASet, error) {
 	hostname = strings.TrimRight(hostname, ".")
 	labels := strings.Split(hostname, ".")
 	// See RFC 6844 "Certification Authority Processing" for pseudocode.
@@ -606,7 +609,7 @@ func (va *ValidationAuthorityImpl) getCAASet(hostname string) (*CAASet, error) {
 		if tld, err := publicsuffix.ICANNTLD(name); err != nil || tld == name {
 			break
 		}
-		CAAs, err := va.DNSResolver.LookupCAA(name)
+		CAAs, err := va.DNSResolver.LookupCAA(ctx, name)
 		if err != nil {
 			return nil, err
 		}
@@ -621,8 +624,13 @@ func (va *ValidationAuthorityImpl) getCAASet(hostname string) (*CAASet, error) {
 // CheckCAARecords verifies that, if the indicated subscriber domain has any CAA
 // records, they authorize the configured CA domain to issue a certificate
 func (va *ValidationAuthorityImpl) CheckCAARecords(identifier core.AcmeIdentifier) (present, valid bool, err error) {
+	// TODO(#1292): add a proper deadline here
+	return va.checkCAARecords(context.TODO(), identifier)
+}
+
+func (va *ValidationAuthorityImpl) checkCAARecords(ctx context.Context, identifier core.AcmeIdentifier) (present, valid bool, err error) {
 	hostname := strings.ToLower(identifier.Value)
-	caaSet, err := va.getCAASet(hostname)
+	caaSet, err := va.getCAASet(ctx, hostname)
 	if err != nil {
 		return
 	}

va/validation-authority_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/go-jose"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/golang.org/x/net/context"
 	"github.com/letsencrypt/boulder/bdns"
 	"github.com/letsencrypt/boulder/metrics"
 	"github.com/letsencrypt/boulder/probs"
@@ -223,7 +224,7 @@ func TestHTTP(t *testing.T) {
 	va := NewValidationAuthorityImpl(&PortConfig{HTTPPort: badPort}, nil, stats, clock.Default())
 	va.DNSResolver = &mocks.DNSResolver{}
 
-	_, prob := va.validateHTTP01(ident, chall)
+	_, prob := va.validateHTTP01(context.Background(), ident, chall)
 	if prob == nil {
 		t.Fatalf("Server's down; expected refusal. Where did we connect?")
 	}
@@ -234,7 +235,7 @@ func TestHTTP(t *testing.T) {
 
 	log.Clear()
 	t.Logf("Trying to validate: %+v\n", chall)
-	_, prob = va.validateHTTP01(ident, chall)
+	_, prob = va.validateHTTP01(context.Background(), ident, chall)
 	if prob != nil {
 		t.Errorf("Unexpected failure in HTTP validation: %s", prob)
 	}
@@ -242,7 +243,7 @@ func TestHTTP(t *testing.T) {
 
 	log.Clear()
 	setChallengeToken(&chall, path404)
-	_, prob = va.validateHTTP01(ident, chall)
+	_, prob = va.validateHTTP01(context.Background(), ident, chall)
 	if prob == nil {
 		t.Fatalf("Should have found a 404 for the challenge.")
 	}
@@ -253,7 +254,7 @@ func TestHTTP(t *testing.T) {
 	setChallengeToken(&chall, pathWrongToken)
 	// The "wrong token" will actually be the expectedToken.  It's wrong
 	// because it doesn't match pathWrongToken.
-	_, prob = va.validateHTTP01(ident, chall)
+	_, prob = va.validateHTTP01(context.Background(), ident, chall)
 	if prob == nil {
 		t.Fatalf("Should have found the wrong token value.")
 	}
@@ -262,7 +263,7 @@ func TestHTTP(t *testing.T) {
 
 	log.Clear()
 	setChallengeToken(&chall, pathMoved)
-	_, prob = va.validateHTTP01(ident, chall)
+	_, prob = va.validateHTTP01(context.Background(), ident, chall)
 	if prob != nil {
 		t.Fatalf("Failed to follow 301 redirect")
 	}
@@ -270,7 +271,7 @@ func TestHTTP(t *testing.T) {
 
 	log.Clear()
 	setChallengeToken(&chall, pathFound)
-	_, prob = va.validateHTTP01(ident, chall)
+	_, prob = va.validateHTTP01(context.Background(), ident, chall)
 	if prob != nil {
 		t.Fatalf("Failed to follow 302 redirect")
 	}
@@ -278,13 +279,13 @@ func TestHTTP(t *testing.T) {
 	test.AssertEquals(t, len(log.GetAllMatching(`redirect from ".*/`+pathMoved+`" to ".*/`+pathValid+`"`)), 1)
 
 	ipIdentifier := core.AcmeIdentifier{Type: core.IdentifierType("ip"), Value: "127.0.0.1"}
-	_, prob = va.validateHTTP01(ipIdentifier, chall)
+	_, prob = va.validateHTTP01(context.Background(), ipIdentifier, chall)
 	if prob == nil {
 		t.Fatalf("IdentifierType IP shouldn't have worked.")
 	}
 	test.AssertEquals(t, prob.Type, probs.MalformedProblem)
 
-	_, prob = va.validateHTTP01(core.AcmeIdentifier{Type: core.IdentifierDNS, Value: "always.invalid"}, chall)
+	_, prob = va.validateHTTP01(context.Background(), core.AcmeIdentifier{Type: core.IdentifierDNS, Value: "always.invalid"}, chall)
 	if prob == nil {
 		t.Fatalf("Domain name is invalid.")
 	}
@@ -292,7 +293,7 @@ func TestHTTP(t *testing.T) {
 
 	setChallengeToken(&chall, pathWaitLong)
 	started := time.Now()
-	_, prob = va.validateHTTP01(ident, chall)
+	_, prob = va.validateHTTP01(context.Background(), ident, chall)
 	took := time.Since(started)
 	// Check that the HTTP connection times out after 5 seconds and doesn't block for 10 seconds
 	test.Assert(t, (took > (time.Second * 5)), "HTTP timed out before 5 seconds")
@@ -318,7 +319,7 @@ func TestHTTPRedirectLookup(t *testing.T) {
 
 	log.Clear()
 	setChallengeToken(&chall, pathMoved)
-	_, prob := va.validateHTTP01(ident, chall)
+	_, prob := va.validateHTTP01(context.Background(), ident, chall)
 	if prob != nil {
 		t.Fatalf("Unexpected failure in redirect (%s): %s", pathMoved, prob)
 	}
@@ -327,7 +328,7 @@ func TestHTTPRedirectLookup(t *testing.T) {
 
 	log.Clear()
 	setChallengeToken(&chall, pathFound)
-	_, prob = va.validateHTTP01(ident, chall)
+	_, prob = va.validateHTTP01(context.Background(), ident, chall)
 	if prob != nil {
 		t.Fatalf("Unexpected failure in redirect (%s): %s", pathFound, prob)
 	}
@@ -337,14 +338,14 @@ func TestHTTPRedirectLookup(t *testing.T) {
 
 	log.Clear()
 	setChallengeToken(&chall, pathReLookupInvalid)
-	_, err = va.validateHTTP01(ident, chall)
+	_, err = va.validateHTTP01(context.Background(), ident, chall)
 	test.AssertError(t, err, chall.Token)
 	test.AssertEquals(t, len(log.GetAllMatching(`Resolved addresses for localhost \[using 127.0.0.1\]: \[127.0.0.1\]`)), 1)
 	test.AssertEquals(t, len(log.GetAllMatching(`No IPv4 addresses found for invalid.invalid`)), 1)
 
 	log.Clear()
 	setChallengeToken(&chall, pathReLookup)
-	_, prob = va.validateHTTP01(ident, chall)
+	_, prob = va.validateHTTP01(context.Background(), ident, chall)
 	if prob != nil {
 		t.Fatalf("Unexpected error in redirect (%s): %s", pathReLookup, prob)
 	}
@@ -354,7 +355,7 @@ func TestHTTPRedirectLookup(t *testing.T) {
 
 	log.Clear()
 	setChallengeToken(&chall, pathRedirectPort)
-	_, err = va.validateHTTP01(ident, chall)
+	_, err = va.validateHTTP01(context.Background(), ident, chall)
 	test.AssertError(t, err, chall.Token)
 	test.AssertEquals(t, len(log.GetAllMatching(`redirect from ".*/port-redirect" to ".*other.valid:8080/path"`)), 1)
 	test.AssertEquals(t, len(log.GetAllMatching(`Resolved addresses for localhost \[using 127.0.0.1\]: \[127.0.0.1\]`)), 1)
@@ -375,7 +376,7 @@ func TestHTTPRedirectLoop(t *testing.T) {
 	va.DNSResolver = &mocks.DNSResolver{}
 
 	log.Clear()
-	_, prob := va.validateHTTP01(ident, chall)
+	_, prob := va.validateHTTP01(context.Background(), ident, chall)
 	if prob == nil {
 		t.Fatalf("Challenge should have failed for %s", chall.Token)
 	}
@@ -396,13 +397,13 @@ func TestHTTPRedirectUserAgent(t *testing.T) {
 	va.UserAgent = rejectUserAgent
 
 	setChallengeToken(&chall, pathMoved)
-	_, prob := va.validateHTTP01(ident, chall)
+	_, prob := va.validateHTTP01(context.Background(), ident, chall)
 	if prob == nil {
 		t.Fatalf("Challenge with rejectUserAgent should have failed (%s).", pathMoved)
 	}
 
 	setChallengeToken(&chall, pathFound)
-	_, prob = va.validateHTTP01(ident, chall)
+	_, prob = va.validateHTTP01(context.Background(), ident, chall)
 	if prob == nil {
 		t.Fatalf("Challenge with rejectUserAgent should have failed (%s).", pathFound)
 	}
@@ -437,14 +438,14 @@ func TestTLSSNI(t *testing.T) {
 	va.DNSResolver = &mocks.DNSResolver{}
 
 	log.Clear()
-	_, prob := va.validateTLSSNI01(ident, chall)
+	_, prob := va.validateTLSSNI01(context.Background(), ident, chall)
 	if prob != nil {
 		t.Fatalf("Unexpected failre in validateTLSSNI01: %s", prob)
 	}
 	test.AssertEquals(t, len(log.GetAllMatching(`Resolved addresses for localhost \[using 127.0.0.1\]: \[127.0.0.1\]`)), 1)
 
 	log.Clear()
-	_, prob = va.validateTLSSNI01(core.AcmeIdentifier{
+	_, prob = va.validateTLSSNI01(context.Background(), core.AcmeIdentifier{
 		Type:  core.IdentifierType("ip"),
 		Value: net.JoinHostPort("127.0.0.1", fmt.Sprintf("%d", port)),
 	}, chall)
@@ -454,7 +455,7 @@ func TestTLSSNI(t *testing.T) {
 	test.AssertEquals(t, prob.Type, probs.MalformedProblem)
 
 	log.Clear()
-	_, prob = va.validateTLSSNI01(core.AcmeIdentifier{Type: core.IdentifierDNS, Value: "always.invalid"}, chall)
+	_, prob = va.validateTLSSNI01(context.Background(), core.AcmeIdentifier{Type: core.IdentifierDNS, Value: "always.invalid"}, chall)
 	if prob == nil {
 		t.Fatalf("Domain name was supposed to be invalid.")
 	}
@@ -467,7 +468,7 @@ func TestTLSSNI(t *testing.T) {
 
 	log.Clear()
 	started := time.Now()
-	_, prob = va.validateTLSSNI01(ident, chall)
+	_, prob = va.validateTLSSNI01(context.Background(), ident, chall)
 	took := time.Since(started)
 	// Check that the HTTP connection times out after 5 seconds and doesn't block for 10 seconds
 	test.Assert(t, (took > (time.Second * 5)), "HTTP timed out before 5 seconds")
@@ -480,7 +481,7 @@ func TestTLSSNI(t *testing.T) {
 
 	// Take down validation server and check that validation fails.
 	hs.Close()
-	_, err = va.validateTLSSNI01(ident, chall)
+	_, err = va.validateTLSSNI01(context.Background(), ident, chall)
 	if err == nil {
 		t.Fatalf("Server's down; expected refusal. Where did we connect?")
 	}
@@ -508,7 +509,7 @@ func TestTLSError(t *testing.T) {
 	va := NewValidationAuthorityImpl(&PortConfig{TLSPort: port}, nil, stats, clock.Default())
 	va.DNSResolver = &mocks.DNSResolver{}
 
-	_, prob := va.validateTLSSNI01(ident, chall)
+	_, prob := va.validateTLSSNI01(context.Background(), ident, chall)
 	if prob == nil {
 		t.Fatalf("TLS validation should have failed: What cert was used?")
 	}
@@ -537,7 +538,7 @@ func TestValidateHTTP(t *testing.T) {
 		Identifier:     ident,
 		Challenges:     []core.Challenge{chall},
 	}
-	va.validate(authz, 0)
+	va.validate(context.Background(), authz, 0)
 
 	test.AssertEquals(t, core.StatusValid, mockRA.lastAuthz.Challenges[0].Status)
 }
@@ -592,7 +593,7 @@ func TestValidateTLSSNI01(t *testing.T) {
 		Identifier:     ident,
 		Challenges:     []core.Challenge{chall},
 	}
-	va.validate(authz, 0)
+	va.validate(context.Background(), authz, 0)
 
 	test.AssertEquals(t, core.StatusValid, mockRA.lastAuthz.Challenges[0].Status)
 }
@@ -614,7 +615,7 @@ func TestValidateTLSSNINotSane(t *testing.T) {
 		Identifier:     ident,
 		Challenges:     []core.Challenge{chall},
 	}
-	va.validate(authz, 0)
+	va.validate(context.Background(), authz, 0)
 
 	test.AssertEquals(t, core.StatusInvalid, mockRA.lastAuthz.Challenges[0].Status)
 }
@@ -651,7 +652,7 @@ func TestCAATimeout(t *testing.T) {
 	va := NewValidationAuthorityImpl(&PortConfig{}, nil, stats, clock.Default())
 	va.DNSResolver = &mocks.DNSResolver{}
 	va.IssuerDomain = "letsencrypt.org"
-	err := va.checkCAA(core.AcmeIdentifier{Type: core.IdentifierDNS, Value: "caa-timeout.com"}, 101)
+	err := va.checkCAA(context.Background(), core.AcmeIdentifier{Type: core.IdentifierDNS, Value: "caa-timeout.com"}, 101)
 	if err.Type != probs.ConnectionProblem {
 		t.Errorf("Expected timeout error type %s, got %s", probs.ConnectionProblem, err.Type)
 	}
@@ -724,7 +725,7 @@ func TestDNSValidationFailure(t *testing.T) {
 		Identifier:     ident,
 		Challenges:     []core.Challenge{chalDNS},
 	}
-	va.validate(authz, 0)
+	va.validate(context.Background(), authz, 0)
 
 	t.Logf("Resulting Authz: %+v", authz)
 	test.AssertNotNil(t, mockRA.lastAuthz, "Should have gotten an authorization")
@@ -753,7 +754,7 @@ func TestDNSValidationInvalid(t *testing.T) {
 	mockRA := &MockRegistrationAuthority{}
 	va.RA = mockRA
 
-	va.validate(authz, 0)
+	va.validate(context.Background(), authz, 0)
 
 	test.AssertNotNil(t, mockRA.lastAuthz, "Should have gotten an authorization")
 	test.Assert(t, authz.Challenges[0].Status == core.StatusInvalid, "Should be invalid.")
@@ -781,7 +782,7 @@ func TestDNSValidationNotSane(t *testing.T) {
 	}
 
 	for i := 0; i < len(authz.Challenges); i++ {
-		va.validate(authz, i)
+		va.validate(context.Background(), authz, i)
 		test.AssertEquals(t, authz.Challenges[i].Status, core.StatusInvalid)
 		test.AssertEquals(t, authz.Challenges[i].Error.Type, probs.MalformedProblem)
 	}
@@ -806,7 +807,7 @@ func TestDNSValidationServFail(t *testing.T) {
 		Identifier:     badIdent,
 		Challenges:     []core.Challenge{chalDNS},
 	}
-	va.validate(authz, 0)
+	va.validate(context.Background(), authz, 0)
 
 	test.AssertNotNil(t, mockRA.lastAuthz, "Should have gotten an authorization")
 	test.Assert(t, authz.Challenges[0].Status == core.StatusInvalid, "Should be invalid.")
@@ -817,7 +818,7 @@ func TestDNSValidationNoServer(t *testing.T) {
 	c, _ := statsd.NewNoopClient()
 	stats := metrics.NewNoopScope()
 	va := NewValidationAuthorityImpl(&PortConfig{}, nil, c, clock.Default())
-	va.DNSResolver = bdns.NewTestDNSResolverImpl(time.Second*5, []string{}, stats)
+	va.DNSResolver = bdns.NewTestDNSResolverImpl(time.Second*5, []string{}, stats, clock.Default(), 1)
 	mockRA := &MockRegistrationAuthority{}
 	va.RA = mockRA
 
@@ -829,7 +830,7 @@ func TestDNSValidationNoServer(t *testing.T) {
 		Identifier:     ident,
 		Challenges:     []core.Challenge{chalDNS},
 	}
-	va.validate(authz, 0)
+	va.validate(context.Background(), authz, 0)
 
 	test.AssertNotNil(t, mockRA.lastAuthz, "Should have gotten an authorization")
 	test.Assert(t, authz.Challenges[0].Status == core.StatusInvalid, "Should be invalid.")
@@ -861,7 +862,7 @@ func TestDNSValidationOK(t *testing.T) {
 		Identifier:     goodIdent,
 		Challenges:     []core.Challenge{chalDNS},
 	}
-	va.validate(authz, 0)
+	va.validate(context.Background(), authz, 0)
 
 	test.AssertNotNil(t, mockRA.lastAuthz, "Should have gotten an authorization")
 	test.Assert(t, authz.Challenges[0].Status == core.StatusValid, "Should be valid.")
@@ -899,7 +900,7 @@ func TestDNSValidationLive(t *testing.T) {
 		Challenges:     []core.Challenge{goodChalDNS},
 	}
 
-	va.validate(authzGood, 0)
+	va.validate(context.Background(), authzGood, 0)
 
 	if authzGood.Challenges[0].Status != core.StatusValid {
 		t.Logf("TestDNSValidationLive on Good did not succeed.")
@@ -916,7 +917,7 @@ func TestDNSValidationLive(t *testing.T) {
 		Challenges:     []core.Challenge{badChalDNS},
 	}
 
-	va.validate(authzBad, 0)
+	va.validate(context.Background(), authzBad, 0)
 	if authzBad.Challenges[0].Status != core.StatusInvalid {
 		t.Logf("TestDNSValidationLive on Bad did succeed inappropriately.")
 	}
@@ -943,7 +944,7 @@ func TestCAAFailure(t *testing.T) {
 		Identifier:     ident,
 		Challenges:     []core.Challenge{chall},
 	}
-	va.validate(authz, 0)
+	va.validate(context.Background(), authz, 0)
 
 	test.AssertEquals(t, core.StatusInvalid, mockRA.lastAuthz.Challenges[0].Status)
 }