Merge branch 'master' into ocsp-decoding

Conflicts: test/amqp-integration-test.py
2015-10-01 17:48:26 -07:00 · 2015-10-01 17:48:26 -07:00 · a0ba72ea35
parent 4d6716ab5a 6dde713f3f
commit a0ba72ea35
34 changed files with 936 additions and 481 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -23,7 +23,6 @@ sudo: false

 services:
  - rabbitmq
-  - mysql

 matrix:
  fast_finish: true
@ -40,37 +39,20 @@ branches:
    - /^test-.*$/

 before_install:
-  # Travis does shallow clones, so there is no master branch present.
-  # But test-no-outdated-migrations.sh needs to check diffs against master.
-  # Fetch just the master branch from origin.
-  - git fetch origin master
-  # Github-PR-Status secret
-  - openssl aes-256-cbc -K $encrypted_53b2630f0fb4_key -iv $encrypted_53b2630f0fb4_iv -in test/github-secret.json.enc -out test/github-secret.json -d || true
+  - source test/travis-before-install.sh

-  - travis_retry go get golang.org/x/tools/cmd/vet
-  - travis_retry go get golang.org/x/tools/cmd/cover
-  - travis_retry go get github.com/golang/lint/golint
-  - travis_retry go get github.com/mattn/goveralls
-  - travis_retry go get github.com/modocache/gover
-  - travis_retry go get github.com/jcjones/github-pr-status
-  - travis_retry go get bitbucket.org/liamstask/goose/cmd/goose
-
-  # Boulder consists of multiple Go packages, which
-  # refer to each other by their absolute GitHub path,
-  # e.g. github.com/letsencrypt/boulder/analysis. That means, by default, if
-  # someone forks the repo, Travis won't pass on their own repo. To fix that,
-  # we add a symlink.
-  - mkdir -p $TRAVIS_BUILD_DIR $GOPATH/src/github.com/letsencrypt
-  - test ! -d $GOPATH/src/github.com/letsencrypt/boulder && ln -s $TRAVIS_BUILD_DIR $GOPATH/src/github.com/letsencrypt/boulder || true
+# Override default Travis install command to prevent it from adding
+# Godeps/_workspace to GOPATH. When that happens, it hides failures that should
+# arise from importing non-vendorized paths.
+install:
+  - true

 env:
  global:
-    - LETSENCRYPT_PATH=/tmp/letsencrypt
+    - LETSENCRYPT_PATH=$HOME/letsencrypt
  matrix:
-    - SKIP_INTEGRATION_TESTS=1
-    - SKIP_UNIT_TESTS=1
-    - RUN_MAKE=1 SKIP_UNIT_TESTS=1 SKIP_INTEGRATION_TESTS=1
+    - RUN="integration vet lint fmt migrations"
+    - RUN="unit"

 script:
-  - bash test/travis_make.sh
  - bash test.sh
--- a/Godeps/Godeps.json
+++ b/Godeps/Godeps.json
@ -98,7 +98,7 @@
 		},
 		{
 			"ImportPath": "github.com/jmhodges/clock",
-			"Rev": "c560df7e034994569e9742f6e7716c173f6286eb"
+			"Rev": "3c4ebd218625c9364c33db6d39c276d80c3090c6"
 		},
 		{
 			"ImportPath": "github.com/letsencrypt/go-jose",
--- a/Godeps/_workspace/src/github.com/jmhodges/clock/.travis.yml
+++ b/Godeps/_workspace/src/github.com/jmhodges/clock/.travis.yml
@ -2,4 +2,7 @@ language: go

 go:
  - 1.3
-  - tip
+  - 1.4
+  - 1.5
+
+sudo: false
--- a/Godeps/_workspace/src/github.com/jmhodges/clock/README.md
+++ b/Godeps/_workspace/src/github.com/jmhodges/clock/README.md
@ -3,8 +3,25 @@ clock

 [![Build Status](https://travis-ci.org/jmhodges/clock.png?branch=master)](https://travis-ci.org/jmhodges/clock)

-A Go package that provides an abstraction for system time that enables
+Package clock provides an abstraction for system time that enables
 testing of time-sensitive code.

+Where you'd use time.Now, instead use clk.Now where clk is an instance
+of Clock.
+
+When running your code in production, pass it a Clock given by
+Default() and when you're running it in your tests, pass it an instance of Clock from NewFake().
+
+When you do that, you can use FakeClock's Add and Set methods to
+control how time behaves in your code making them more reliable while
+also expanding the space of problems you can test.
+
+This code intentionally does not attempt to provide an abstraction
+over time.Ticker and time.Timer because Go does not have the runtime
+or API hooks available to do reliably. See
+https://github.com/golang/go/issues/8869
+
+Be sure to test Time equality with time.Time#Equal, not ==.
+
 For documentation, see the
 [godoc](http://godoc.org/github.com/jmhodges/clock).
--- a/Godeps/_workspace/src/github.com/jmhodges/clock/clock.go
+++ b/Godeps/_workspace/src/github.com/jmhodges/clock/clock.go
@ -1,9 +1,21 @@
 // Package clock provides an abstraction for system time that enables
 // testing of time-sensitive code.
 //
-// By passing in Default to production code, you can then use NewFake
-// in tests to create Clocks that control what time the production
-// code sees.
+// Where you'd use time.Now, instead use clk.Now where clk is an
+// instance of Clock.
+//
+// When running your code in production, pass it a Clock given by
+// Default() and when you're running it in your tests, pass it an
+// instance of Clock from NewFake().
+//
+// When you do that, you can use FakeClock's Add and Set methods to
+// control how time behaves in your code making them more reliable
+// while also expanding the space of problems you can test.
+//
+// This code intentionally does not attempt to provide an abstraction
+// over time.Ticker and time.Timer because Go does not have the
+// runtime or API hooks available to do reliably. See
+// https://github.com/golang/go/issues/8869
 //
 // Be sure to test Time equality with time.Time#Equal, not ==.
 package clock
@ -29,6 +41,7 @@ type Clock interface {
 	// Now returns the Clock's current view of the time. Mutating the
 	// returned Time will not mutate the clock's time.
 	Now() time.Time
+	Sleep(time.Duration)
 }

 type sysClock struct{}
@ -37,6 +50,10 @@ func (s sysClock) Now() time.Time {
 	return time.Now()
 }

+func (s sysClock) Sleep(d time.Duration) {
+	time.Sleep(d)
+}
+
 // NewFake returns a FakeClock to be used in tests that need to
 // manipulate time. Its initial value is always the unix epoch in the
 // UTC timezone. The FakeClock returned is thread-safe.
@ -54,6 +71,9 @@ type FakeClock interface {
 	Clock
 	// Adjust the time that will be returned by Now.
 	Add(d time.Duration)
+
+	// Set the Clock's time to exactly the time given.
+	Set(t time.Time)
 }

 // To prevent mistakes with the API, we hide this behind NewFake. It's
@ -71,8 +91,22 @@ func (f *fake) Now() time.Time {
 	return f.t
 }

+func (f *fake) Sleep(d time.Duration) {
+	if d < 0 {
+		// time.Sleep just returns immediately. Do the same.
+		return
+	}
+	f.Add(d)
+}
+
 func (f *fake) Add(d time.Duration) {
 	f.Lock()
 	defer f.Unlock()
 	f.t = f.t.Add(d)
 }
+
+func (f *fake) Set(t time.Time) {
+	f.Lock()
+	defer f.Unlock()
+	f.t = t
+}
--- a/Godeps/_workspace/src/github.com/jmhodges/clock/clock_test.go
+++ b/Godeps/_workspace/src/github.com/jmhodges/clock/clock_test.go
@ -9,6 +9,8 @@ import (
 func TestFakeClockGoldenPath(t *testing.T) {
 	clk := NewFake()
 	second := NewFake()
+	oldT := clk.Now()
+
 	if !clk.Now().Equal(second.Now()) {
 		t.Errorf("clocks must start out at the same time but didn't: %#v vs %#v", clk.Now(), second.Now())
 	}
@ -16,6 +18,27 @@ func TestFakeClockGoldenPath(t *testing.T) {
 	if clk.Now().Equal(second.Now()) {
 		t.Errorf("clocks different must differ: %#v vs %#v", clk.Now(), second.Now())
 	}
+
+	clk.Set(oldT)
+	if !clk.Now().Equal(second.Now()) {
+		t.Errorf("clk should have been been set backwards: %#v vs %#v", clk.Now(), second.Now())
+	}
+
+	clk.Sleep(time.Second)
+	if clk.Now().Equal(second.Now()) {
+		t.Errorf("clk should have been set forwards (by sleeping): %#v vs %#v", clk.Now(), second.Now())
+	}
+}
+
+func TestNegativeSleep(t *testing.T) {
+	clk := NewFake()
+	clk.Add(1 * time.Hour)
+	first := clk.Now()
+	clk.Sleep(-10 * time.Second)
+	if !clk.Now().Equal(first) {
+		t.Errorf("clk should not move in time on a negative sleep")
+	}
+
 }

 func ExampleClock() {
--- a/29
+++ b/29
@ -9,8 +9,12 @@ VERSION ?= 1.0.0
 EPOCH ?= 1
 MAINTAINER ?= "Community"

-CMD_OBJECTS = $(shell find ./cmd -maxdepth 1 -mindepth 1 -type d -exec basename '{}' \;)
-OBJECTS = $(CMD_OBJECTS) pkcs11bench
+CMDS = $(shell find ./cmd -maxdepth 1 -mindepth 1 -type d)
+CMD_BASENAMES = $(shell echo $(CMDS) | xargs -n1 basename)
+CMD_BINS = $(addprefix $(OBJDIR)/, $(CMD_BASENAMES) )
+OBJECTS = $(CMD_BINS) $(OBJDIR)/pkcs11bench
+
+GO_BUILD_FLAGS =

 # Build environment variables (referencing core/util.go)
 COMMIT_ID = $(shell git rev-parse --short HEAD)
@ -29,16 +33,15 @@ all: build

 build: $(OBJECTS)

-pre:
+$(OBJDIR):
 	@mkdir -p $(OBJDIR)

-# Compile each of the binaries
-$(CMD_OBJECTS): pre
-	@echo [go] bin/$@
-	@go build -o ./bin/$@ -ldflags \
+$(CMD_BINS): build_cmds
+
+build_cmds: | $(OBJDIR)
+	GOBIN=$(OBJDIR) go install $(GO_BUILD_FLAGS) -ldflags \
 		"-X \"$(BUILD_ID_VAR)=$(BUILD_ID)\" -X \"$(BUILD_TIME_VAR)=$(BUILD_TIME)\" \
-		-X \"$(BUILD_HOST_VAR)=$(BUILD_HOST)\"" \
-		./cmd/$@/
+		-X \"$(BUILD_HOST_VAR)=$(BUILD_HOST)\"" ./...

 clean:
 	rm -f $(OBJDIR)/*
@ -64,7 +67,7 @@ archive:
 # Version and Epoch, such as:
 #
 # VERSION=0.1.9 EPOCH=52 MAINTAINER="$(whoami)" ARCHIVEDIR=/tmp make build rpm
-rpm:
+rpm: build
 	fpm -s dir -t rpm --rpm-digest sha256 --name "boulder" \
 		--license "Mozilla Public License v2.0" --vendor "ISRG" \
 		--url "https://github.com/letsencrypt/boulder" --prefix=/opt/boulder \
@ -72,7 +75,7 @@ rpm:
 		--package $(ARCHIVEDIR)/boulder-$(VERSION)-$(COMMIT_ID).x86_64.rpm \
 		--description "Boulder is an ACME-compatible X.509 Certificate Authority" \
 		--depends "libtool-ltdl" --maintainer "$(MAINTAINER)" \
-		test/boulder-config.json sa/_db ca/_db $(foreach var,$(OBJECTS), $(OBJDIR)/$(var))
+		test/boulder-config.json sa/_db $(OBJECTS)

-pkcs11bench: pre
-	go test -o ./bin/pkcs11bench -c ./Godeps/_workspace/src/github.com/cloudflare/cfssl/crypto/pkcs11key/
+$(OBJDIR)/pkcs11bench: ./Godeps/_workspace/src/github.com/cloudflare/cfssl/crypto/pkcs11key/*.go | $(OBJDIR)
+	go test -o $(OBJDIR)/pkcs11bench -c ./Godeps/_workspace/src/github.com/cloudflare/cfssl/crypto/pkcs11key/
--- a/ca/certificate-authority.go
+++ b/ca/certificate-authority.go
@ -376,34 +376,12 @@ func (ca *CertificateAuthorityImpl) IssueCertificate(csr x509.CertificateRequest
 		return emptyCert, err
 	}

-	// Attempt to generate the OCSP Response now. If this raises an error, it is
-	// logged but is not returned to the caller, as an error at this point does
-	// not constitute an issuance failure.
+	// Submit the certificate to any configured CT logs
 	certObj, err := x509.ParseCertificate(certDER)
 	if err != nil {
 		ca.log.Warning(fmt.Sprintf("Post-Issuance OCSP failed parsing Certificate: %s", err))
 		return cert, nil
 	}
-
-	serial := core.SerialToString(certObj.SerialNumber)
-	signRequest := ocsp.SignRequest{
-		Certificate: certObj,
-		Status:      string(core.OCSPStatusGood),
-	}
-
-	ocspResponse, err := ca.OCSPSigner.Sign(signRequest)
-	if err != nil {
-		ca.log.Warning(fmt.Sprintf("Post-Issuance OCSP failed signing: %s", err))
-		return cert, nil
-	}
-
-	err = ca.SA.UpdateOCSP(serial, ocspResponse)
-	if err != nil {
-		ca.log.Warning(fmt.Sprintf("Post-Issuance OCSP failed storing: %s", err))
-		return cert, nil
-	}
-
-	// Submit the certificate to any configured CT logs
 	go ca.Publisher.SubmitToCT(certObj.Raw)

 	// Do not return an err at this point; caller must know that the Certificate
--- a/cmd/boulder-wfe/main.go
+++ b/cmd/boulder-wfe/main.go
@ -13,6 +13,7 @@ import (
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/codegangsta/cli"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/facebookgo/httpdown"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/streadway/amqp"

 	"github.com/letsencrypt/boulder/cmd"
@ -125,14 +126,14 @@ func main() {

 		auditlogger.Info(fmt.Sprintf("Server running, listening on %s...\n", c.WFE.ListenAddress))
 		srv := &http.Server{
-			Addr:      c.WFE.ListenAddress,
-			ConnState: httpMonitor.ConnectionMonitor,
-			Handler:   httpMonitor.Handle(),
+			Addr:    c.WFE.ListenAddress,
+			Handler: httpMonitor.Handle(),
 		}

 		hd := &httpdown.HTTP{
 			StopTimeout: wfe.ShutdownStopTimeout,
 			KillTimeout: wfe.ShutdownKillTimeout,
+			Stats:       metrics.NewFBAdapter(stats, "WFE", clock.Default()),
 		}
 		err = httpdown.ListenAndServe(srv, hd)
 		cmd.FailOnError(err, "Error starting HTTP server")
--- a/cmd/expiration-mailer/main_test.go
+++ b/cmd/expiration-mailer/main_test.go
@ -63,7 +63,7 @@ func (f fakeRegStore) GetRegistration(id int64) (core.Registration, error) {
 	r, ok := f.RegById[id]
 	if !ok {
 		msg := fmt.Sprintf("no such registration %d", id)
-		return r, sa.NoSuchRegistrationError{Msg: msg}
+		return r, core.NoSuchRegistrationError(msg)
 	}
 	return r, nil
 }
--- a/cmd/ocsp-responder/main.go
+++ b/cmd/ocsp-responder/main.go
@ -17,6 +17,7 @@ import (
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
 	cfocsp "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cloudflare/cfssl/ocsp"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/facebookgo/httpdown"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/golang.org/x/crypto/ocsp"
 	gorp "github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1"
 	"github.com/letsencrypt/boulder/metrics"
@ -172,14 +173,14 @@ func main() {

 		httpMonitor := metrics.NewHTTPMonitor(stats, m, "OCSP")
 		srv := &http.Server{
-			Addr:      c.OCSPResponder.ListenAddress,
-			ConnState: httpMonitor.ConnectionMonitor,
-			Handler:   httpMonitor.Handle(),
+			Addr:    c.OCSPResponder.ListenAddress,
+			Handler: httpMonitor.Handle(),
 		}

 		hd := &httpdown.HTTP{
 			StopTimeout: stopTimeout,
 			KillTimeout: killTimeout,
+			Stats:       metrics.NewFBAdapter(stats, "OCSP", clock.Default()),
 		}
 		err = httpdown.ListenAndServe(srv, hd)
 		cmd.FailOnError(err, "Error starting HTTP server")
--- a/cmd/ocsp-updater/main.go
+++ b/cmd/ocsp-updater/main.go
@ -9,10 +9,11 @@ import (
 	"crypto/x509"
 	"database/sql"
 	"fmt"
+	"os"
 	"time"

 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
-	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/codegangsta/cli"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/streadway/amqp"
 	gorp "github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1"

@ -23,20 +24,70 @@ import (
 	"github.com/letsencrypt/boulder/sa"
 )

-// FatalError indicates the updater should stop execution
-type FatalError string
-
-func (e FatalError) Error() string { return string(e) }
-
 // OCSPUpdater contains the useful objects for the Updater
 type OCSPUpdater struct {
 	stats statsd.Statter
 	log   *blog.AuditLogger
-	cac   rpc.CertificateAuthorityClient
+	clk   clock.Clock
+
 	dbMap *gorp.DbMap
+
+	cac core.CertificateAuthority
+
+	// Bits various loops need but don't really fit in the looper struct
+	ocspMinTimeToExpiry time.Duration
+
+	newCertificatesLoop  *looper
+	oldOCSPResponsesLoop *looper
 }

-func setupClients(c cmd.Config, stats statsd.Statter) (rpc.CertificateAuthorityClient, chan *amqp.Error) {
+// This is somewhat gross but can be pared down a bit once the publisher and this
+// are fully smooshed together
+func newUpdater(
+	log *blog.AuditLogger,
+	stats statsd.Statter,
+	clk clock.Clock,
+	dbMap *gorp.DbMap,
+	ca core.CertificateAuthority,
+	config cmd.OCSPUpdaterConfig,
+) (*OCSPUpdater, error) {
+	if config.NewCertificateBatchSize == 0 ||
+		config.OldOCSPBatchSize == 0 {
+		return nil, fmt.Errorf("Batch sizes must be non-zero")
+	}
+
+	updater := OCSPUpdater{
+		stats: stats,
+		log:   log,
+		clk:   clk,
+		dbMap: dbMap,
+		cac:   ca,
+	}
+
+	// Setup loops
+	updater.newCertificatesLoop = &looper{
+		clk:       clk,
+		stats:     stats,
+		batchSize: config.NewCertificateBatchSize,
+		tickDur:   config.NewCertificateWindow.Duration,
+		tickFunc:  updater.newCertificateTick,
+		name:      "NewCertificates",
+	}
+	updater.oldOCSPResponsesLoop = &looper{
+		clk:       clk,
+		stats:     stats,
+		batchSize: config.OldOCSPBatchSize,
+		tickDur:   config.OldOCSPWindow.Duration,
+		tickFunc:  updater.oldOCSPResponsesTick,
+		name:      "OldOCSPResponses",
+	}
+
+	updater.ocspMinTimeToExpiry = config.OCSPMinTimeToExpiry.Duration
+
+	return &updater, nil
+}
+
+func setupClients(c cmd.Config, stats statsd.Statter) (core.CertificateAuthority, chan *amqp.Error) {
 	ch, err := rpc.AmqpChannel(c)
 	cmd.FailOnError(err, "Could not connect to AMQP")

@ -51,28 +102,65 @@ func setupClients(c cmd.Config, stats statsd.Statter) (rpc.CertificateAuthorityC
 	return cac, closeChan
 }

-func (updater *OCSPUpdater) processResponse(tx *gorp.Transaction, serial string) error {
-	certObj, err := tx.Get(core.Certificate{}, serial)
-	if err != nil {
-		return err
-	}
-	statusObj, err := tx.Get(core.CertificateStatus{}, serial)
-	if err != nil {
-		return err
+func (updater *OCSPUpdater) findStaleOCSPResponses(oldestLastUpdatedTime time.Time, batchSize int) ([]core.CertificateStatus, error) {
+	var statuses []core.CertificateStatus
+	_, err := updater.dbMap.Select(
+		&statuses,
+		`SELECT cs.*
+		 FROM certificateStatus AS cs
+		 JOIN certificates AS cert
+		 ON cs.serial = cert.serial
+		 WHERE cs.ocspLastUpdated < :lastUpdate
+		 AND cert.expires > now()
+		 ORDER BY cs.ocspLastUpdated ASC
+		 LIMIT :limit`,
+		map[string]interface{}{
+			"lastUpdate": oldestLastUpdatedTime,
+			"limit":      batchSize,
+		},
+	)
+	if err == sql.ErrNoRows {
+		return statuses, nil
 	}
+	return statuses, err
+}

-	cert, ok := certObj.(*core.Certificate)
-	if !ok {
-		return fmt.Errorf("Cast failure")
+func (updater *OCSPUpdater) getCertificatesWithMissingResponses(batchSize int) ([]core.CertificateStatus, error) {
+	var statuses []core.CertificateStatus
+	_, err := updater.dbMap.Select(
+		&statuses,
+		`SELECT * FROM certificateStatus
+		 WHERE ocspLastUpdated = 0
+		 LIMIT :limit`,
+		map[string]interface{}{
+			"limit": batchSize,
+		},
+	)
+	if err == sql.ErrNoRows {
+		return statuses, nil
 	}
-	status, ok := statusObj.(*core.CertificateStatus)
-	if !ok {
-		return fmt.Errorf("Cast failure")
+	return statuses, err
+}
+
+type responseMeta struct {
+	*core.OCSPResponse
+	*core.CertificateStatus
+}
+
+func (updater *OCSPUpdater) generateResponse(status core.CertificateStatus) (responseMeta, error) {
+	var cert core.Certificate
+	err := updater.dbMap.SelectOne(
+		&cert,
+		"SELECT * FROM certificates WHERE serial = :serial",
+		map[string]interface{}{"serial": status.Serial},
+	)
+	if err != nil {
+		return responseMeta{}, err
 	}

 	_, err = x509.ParseCertificate(cert.DER)
 	if err != nil {
-		return err
+		return responseMeta{}, err
 	}

 	signRequest := core.OCSPSigningRequest{
@ -84,21 +172,28 @@ func (updater *OCSPUpdater) processResponse(tx *gorp.Transaction, serial string)

 	ocspResponse, err := updater.cac.GenerateOCSP(signRequest)
 	if err != nil {
-		return err
+		return responseMeta{}, err
 	}

-	timeStamp := time.Now()
+	timestamp := updater.clk.Now()
+	status.OCSPLastUpdated = timestamp
+	ocspResp := &core.OCSPResponse{
+		Serial:    cert.Serial,
+		CreatedAt: timestamp,
+		Response:  ocspResponse,
+	}
+	return responseMeta{ocspResp, &status}, nil
+}

+func (updater *OCSPUpdater) storeResponse(tx *gorp.Transaction, meta responseMeta) error {
 	// Record the response.
-	ocspResp := &core.OCSPResponse{Serial: serial, CreatedAt: timeStamp, Response: ocspResponse}
-	err = tx.Insert(ocspResp)
+	err := tx.Insert(meta.OCSPResponse)
 	if err != nil {
 		return err
 	}

 	// Reset the update clock
-	status.OCSPLastUpdated = timeStamp
-	_, err = tx.Update(status)
+	_, err = tx.Update(meta.CertificateStatus)
 	if err != nil {
 		return err
 	}
@ -107,97 +202,101 @@ func (updater *OCSPUpdater) processResponse(tx *gorp.Transaction, serial string)
 	return nil
 }

-// Produce one OCSP response for the given serial, returning err
-// if anything went wrong. This method will open and commit a transaction.
-func (updater *OCSPUpdater) updateOneSerial(serial string) error {
-	innerStart := time.Now()
-	// Each response gets a transaction. In the future we can increase
-	// performance by batching transactions.
-	// The key thing to think through is the cost of rollbacks, and whether
-	// we should rollback if CA/HSM fails to sign the response or only
-	// upon a partial DB insert.
-	tx, err := updater.dbMap.Begin()
+// newCertificateTick checks for certificates issued since the last tick and
+// generates and stores OCSP responses for these certs
+func (updater *OCSPUpdater) newCertificateTick(batchSize int) {
+	// Check for anything issued between now and previous tick and generate first
+	// OCSP responses
+	statuses, err := updater.getCertificatesWithMissingResponses(batchSize)
 	if err != nil {
-		updater.log.Err(fmt.Sprintf("OCSP %s: Error starting transaction, aborting: %s", serial, err))
-		updater.stats.Inc("OCSP.Updates.Failed", 1, 1.0)
-		tx.Rollback()
-		// Failure to begin transaction is a fatal error.
-		return FatalError(err.Error())
+		return
 	}

-	if err := updater.processResponse(tx, serial); err != nil {
-		updater.log.Err(fmt.Sprintf("OCSP %s: Could not process OCSP Response, skipping: %s", serial, err))
-		updater.stats.Inc("OCSP.Updates.Failed", 1, 1.0)
-		tx.Rollback()
-		return err
-	}
-
-	err = tx.Commit()
-	if err != nil {
-		updater.log.Err(fmt.Sprintf("OCSP %s: Error committing transaction, skipping: %s", serial, err))
-		updater.stats.Inc("OCSP.Updates.Failed", 1, 1.0)
-		tx.Rollback()
-		return err
-	}
-
-	updater.log.Info(fmt.Sprintf("OCSP %s: OK", serial))
-	updater.stats.Inc("OCSP.Updates.Processed", 1, 1.0)
-	updater.stats.TimingDuration("OCSP.Updates.UpdateLatency", time.Since(innerStart), 1.0)
-	return nil
+	updater.generateOCSPResponses(statuses)
 }

-// findStaleResponses opens a transaction and processes up to responseLimit
-// responses in a single batch. The responseLimit should be relatively small,
-// so as to limit the chance of the transaction failing due to concurrent
-// updates.
-func (updater *OCSPUpdater) findStaleResponses(oldestLastUpdatedTime time.Time, responseLimit int) error {
-	var certificateStatus []core.CertificateStatus
-	_, err := updater.dbMap.Select(&certificateStatus,
-		`SELECT cs.* FROM certificateStatus AS cs JOIN certificates AS cert ON cs.serial = cert.serial
-		 WHERE cs.ocspLastUpdated < ? AND cert.expires > now()
-		 ORDER BY cs.ocspLastUpdated ASC
-		 LIMIT ?`, oldestLastUpdatedTime, responseLimit)
-
-	if err == sql.ErrNoRows {
-		updater.log.Info("All up to date. No OCSP responses needed.")
-	} else if err != nil {
-		updater.log.Err(fmt.Sprintf("Error loading certificate status: %s", err))
-	} else {
-		updater.log.Info(fmt.Sprintf("Processing OCSP Responses...\n"))
-		outerStart := time.Now()
-
-		for i, status := range certificateStatus {
-			updater.log.Debug(fmt.Sprintf("OCSP %s: (%d/%d)", status.Serial, i, responseLimit))
-
-			err = updater.updateOneSerial(status.Serial)
-			// Abort if we recieve a fatal error
-			if _, ok := err.(FatalError); ok {
-				return err
-			}
+func (updater *OCSPUpdater) generateOCSPResponses(statuses []core.CertificateStatus) {
+	responses := []responseMeta{}
+	for _, status := range statuses {
+		meta, err := updater.generateResponse(status)
+		if err != nil {
+			updater.log.AuditErr(fmt.Errorf("Failed to generate OCSP response: %s", err))
+			updater.stats.Inc("OCSP.Errors.ResponseGeneration", 1, 1.0)
+			continue
 		}
-
-		updater.stats.TimingDuration("OCSP.Updates.BatchLatency", time.Since(outerStart), 1.0)
-		updater.stats.Inc("OCSP.Updates.BatchesProcessed", 1, 1.0)
+		responses = append(responses, meta)
+		updater.stats.Inc("OCSP.GeneratedResponses", 1, 1.0)
 	}

-	return err
+	tx, err := updater.dbMap.Begin()
+	if err != nil {
+		updater.log.AuditErr(fmt.Errorf("Failed to open OCSP response transaction: %s", err))
+		updater.stats.Inc("OCSP.Errors.OpenTransaction", 1, 1.0)
+		return
+	}
+	for _, meta := range responses {
+		err = updater.storeResponse(tx, meta)
+		if err != nil {
+			updater.log.AuditErr(fmt.Errorf("Failed to store OCSP response: %s", err))
+			updater.stats.Inc("OCSP.Errors.StoreResponse", 1, 1.0)
+			tx.Rollback()
+			return
+		}
+	}
+	err = tx.Commit()
+	if err != nil {
+		updater.log.AuditErr(fmt.Errorf("Failed to commit OCSP response transaction: %s", err))
+		updater.stats.Inc("OCSP.Errors.CommitTransaction", 1, 1.0)
+		return
+	}
+	updater.stats.Inc("OCSP.StoredResponses", int64(len(responses)), 1.0)
+
+	return
+}
+
+// oldOCSPResponsesTick looks for certificates with stale OCSP responses and
+// generates/stores new ones
+func (updater *OCSPUpdater) oldOCSPResponsesTick(batchSize int) {
+	now := time.Now()
+	statuses, err := updater.findStaleOCSPResponses(now.Add(-updater.ocspMinTimeToExpiry), batchSize)
+	if err != nil {
+		updater.stats.Inc("OCSP.Errors.FindStaleResponses", 1, 1.0)
+		updater.log.AuditErr(fmt.Errorf("Failed to find stale OCSP responses: %s", err))
+		return
+	}
+
+	updater.generateOCSPResponses(statuses)
+}
+
+type looper struct {
+	clk       clock.Clock
+	stats     statsd.Statter
+	batchSize int
+	tickDur   time.Duration
+	tickFunc  func(int)
+	name      string
+}
+
+func (l *looper) loop() {
+	for {
+		tickStart := l.clk.Now()
+		l.tickFunc(l.batchSize)
+		l.stats.TimingDuration(fmt.Sprintf("OCSP.%s.TickDuration", l.name), time.Since(tickStart), 1.0)
+		l.stats.Inc(fmt.Sprintf("OCSP.%s.Ticks", l.name), 1, 1.0)
+		tickEnd := tickStart.Add(time.Since(tickStart))
+		expectedTickEnd := tickStart.Add(l.tickDur)
+		if tickEnd.After(expectedTickEnd) {
+			l.stats.Inc(fmt.Sprintf("OCSP.%s.LongTicks", l.name), 1, 1.0)
+		}
+		// Sleep for the remaining tick period (if this is a negative number sleep
+		// will not do anything and carry on)
+		l.clk.Sleep(expectedTickEnd.Sub(tickEnd))
+	}
 }

 func main() {
 	app := cmd.NewAppShell("ocsp-updater", "Generates and updates OCSP responses")

-	app.App.Flags = append(app.App.Flags, cli.IntFlag{
-		Name:   "limit",
-		Value:  100,
-		EnvVar: "OCSP_LIMIT",
-		Usage:  "Count of responses to process per run",
-	})
-
-	app.Config = func(c *cli.Context, config cmd.Config) cmd.Config {
-		config.OCSPUpdater.ResponseLimit = c.GlobalInt("limit")
-		return config
-	}
-
 	app.Action = func(c cmd.Config) {
 		// Set up logging
 		stats, err := statsd.NewClient(c.Statsd.Server, c.Statsd.Prefix)
@ -207,12 +306,13 @@ func main() {
 		cmd.FailOnError(err, "Could not connect to Syslog")
 		auditlogger.Info(app.VersionString())

+		blog.SetAuditLogger(auditlogger)
+
 		// AUDIT[ Error Conditions ] 9cc4d537-8534-4970-8665-4b382abe82f3
 		defer auditlogger.AuditPanic()

-		blog.SetAuditLogger(auditlogger)
-
 		go cmd.DebugServer(c.OCSPUpdater.DebugAddr)
+		go cmd.ProfileCmd("OCSP-Updater", stats)

 		// Configure DB
 		dbMap, err := sa.NewDbMap(c.OCSPUpdater.DBConnect)
@ -220,40 +320,28 @@ func main() {

 		cac, closeChan := setupClients(c, stats)

-		go func() {
-			// Abort if we disconnect from AMQP
-			for {
-				for err := range closeChan {
-					auditlogger.Warning(fmt.Sprintf(" [!] AMQP Channel closed, aborting early: [%s]", err))
-					panic(err)
-				}
-			}
-		}()
+		updater, err := newUpdater(
+			auditlogger,
+			stats,
+			clock.Default(),
+			dbMap,
+			cac,
+			// Necessary evil for now
+			c.OCSPUpdater,
+		)

-		updater := &OCSPUpdater{
-			cac:   cac,
-			dbMap: dbMap,
-			stats: stats,
-			log:   auditlogger,
-		}
+		go updater.newCertificatesLoop.loop()
+		go updater.oldOCSPResponsesLoop.loop()

-		// Calculate the cut-off timestamp
-		if c.OCSPUpdater.MinTimeToExpiry == "" {
-			panic("Config must specify a MinTimeToExpiry period.")
-		}
-		dur, err := time.ParseDuration(c.OCSPUpdater.MinTimeToExpiry)
-		cmd.FailOnError(err, "Could not parse MinTimeToExpiry from config.")
+		cmd.FailOnError(err, "Failed to create updater")

-		oldestLastUpdatedTime := time.Now().Add(-dur)
-		auditlogger.Info(fmt.Sprintf("Searching for OCSP responses older than %s", oldestLastUpdatedTime))
-
-		// When we choose to batch responses, it may be best to restrict count here,
-		// change the transaction to survive the whole findStaleResponses, and to
-		// loop this method call however many times is appropriate.
-		err = updater.findStaleResponses(oldestLastUpdatedTime, c.OCSPUpdater.ResponseLimit)
-		if err != nil {
-			auditlogger.WarningErr(err)
-		}
+		// TODO(): When the channel falls over so do we for now, if the AMQP channel
+		// has already closed there is no real cleanup we can do. This is due to
+		// really needing to change the underlying AMQP Server/Client reconnection
+		// logic.
+		err = <-closeChan
+		auditlogger.AuditErr(fmt.Errorf(" [!] AMQP Channel closed, exiting: [%s]", err))
+		os.Exit(1)
 	}

 	app.Run()
--- a/cmd/ocsp-updater/main_test.go
+++ b/cmd/ocsp-updater/main_test.go
@ -1 +1,197 @@
 package main
+
+import (
+	"crypto/x509"
+	"testing"
+	"time"
+
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/gopkg.in/gorp.v1"
+	"github.com/letsencrypt/boulder/core"
+
+	"github.com/letsencrypt/boulder/sa"
+	"github.com/letsencrypt/boulder/sa/satest"
+	"github.com/letsencrypt/boulder/test"
+)
+
+type mockCA struct{}
+
+func (ca *mockCA) IssueCertificate(csr x509.CertificateRequest, regID int64) (core.Certificate, error) {
+	return core.Certificate{}, nil
+}
+
+func (ca *mockCA) GenerateOCSP(xferObj core.OCSPSigningRequest) (ocsp []byte, err error) {
+	ocsp = []byte{1, 2, 3}
+	return
+}
+
+func (ca *mockCA) RevokeCertificate(serial string, reasonCode core.RevocationCode) (err error) {
+	return
+}
+
+const dbConnStr = "mysql+tcp://boulder@localhost:3306/boulder_sa_test"
+
+func setup(t *testing.T) (OCSPUpdater, core.StorageAuthority, *gorp.DbMap, clock.FakeClock, func()) {
+	dbMap, err := sa.NewDbMap(dbConnStr)
+	test.AssertNotError(t, err, "Failed to create dbMap")
+
+	fc := clock.NewFake()
+	fc.Add(1 * time.Hour)
+
+	sa, err := sa.NewSQLStorageAuthority(dbMap, fc)
+	test.AssertNotError(t, err, "Failed to create SA")
+
+	cleanUp := test.ResetTestDatabase(t, dbMap.Db)
+
+	stats, _ := statsd.NewNoopClient(nil)
+
+	updater := OCSPUpdater{
+		dbMap: dbMap,
+		clk:   fc,
+		cac:   &mockCA{},
+		stats: stats,
+	}
+
+	return updater, sa, dbMap, fc, cleanUp
+}
+
+func TestGenerateAndStoreOCSPResponse(t *testing.T) {
+	updater, sa, dbMap, _, cleanUp := setup(t)
+	defer cleanUp()
+
+	reg := satest.CreateWorkingRegistration(t, sa)
+	parsedCert, err := core.LoadCert("test-cert.pem")
+	test.AssertNotError(t, err, "Couldn't read test certificate")
+	_, err = sa.AddCertificate(parsedCert.Raw, reg.ID)
+	test.AssertNotError(t, err, "Couldn't add www.eff.org.der")
+
+	status, err := sa.GetCertificateStatus(core.SerialToString(parsedCert.SerialNumber))
+	test.AssertNotError(t, err, "Couldn't get the core.CertificateStatus from the database")
+
+	meta, err := updater.generateResponse(status)
+	test.AssertNotError(t, err, "Couldn't generate OCSP response")
+	tx, err := dbMap.Begin()
+	test.AssertNotError(t, err, "Couldn't open a transaction")
+	err = updater.storeResponse(tx, meta)
+	test.AssertNotError(t, err, "Couldn't store OCSP response")
+	err = tx.Commit()
+	test.AssertNotError(t, err, "Couldn't close transaction")
+
+	var ocspResponse core.OCSPResponse
+	err = dbMap.SelectOne(
+		&ocspResponse,
+		"SELECT * from ocspResponses WHERE serial = :serial ORDER BY id DESC LIMIT 1;",
+		map[string]interface{}{"serial": status.Serial},
+	)
+	test.AssertNotError(t, err, "Couldn't get OCSP response from database")
+}
+
+func TestGenerateOCSPResponses(t *testing.T) {
+	updater, sa, _, fc, cleanUp := setup(t)
+	defer cleanUp()
+
+	reg := satest.CreateWorkingRegistration(t, sa)
+	parsedCert, err := core.LoadCert("test-cert.pem")
+	test.AssertNotError(t, err, "Couldn't read test certificate")
+	_, err = sa.AddCertificate(parsedCert.Raw, reg.ID)
+	test.AssertNotError(t, err, "Couldn't add test-cert.pem")
+	parsedCert, err = core.LoadCert("test-cert-b.pem")
+	test.AssertNotError(t, err, "Couldn't read test certificate")
+	_, err = sa.AddCertificate(parsedCert.Raw, reg.ID)
+	test.AssertNotError(t, err, "Couldn't add test-cert-b.pem")
+
+	earliest := fc.Now().Add(-time.Hour)
+	certs, err := updater.findStaleOCSPResponses(earliest, 10)
+	test.AssertNotError(t, err, "Couldn't find stale responses")
+	test.AssertEquals(t, len(certs), 2)
+
+	updater.generateOCSPResponses(certs)
+
+	certs, err = updater.findStaleOCSPResponses(earliest, 10)
+	test.AssertNotError(t, err, "Failed to find stale responses")
+	test.AssertEquals(t, len(certs), 0)
+}
+
+func TestFindStaleOCSPResponses(t *testing.T) {
+	updater, sa, dbMap, fc, cleanUp := setup(t)
+	defer cleanUp()
+
+	reg := satest.CreateWorkingRegistration(t, sa)
+	parsedCert, err := core.LoadCert("test-cert.pem")
+	test.AssertNotError(t, err, "Couldn't read test certificate")
+	_, err = sa.AddCertificate(parsedCert.Raw, reg.ID)
+	test.AssertNotError(t, err, "Couldn't add www.eff.org.der")
+
+	earliest := fc.Now().Add(-time.Hour)
+	certs, err := updater.findStaleOCSPResponses(earliest, 10)
+	test.AssertNotError(t, err, "Couldn't find certificate")
+	test.AssertEquals(t, len(certs), 1)
+
+	status, err := sa.GetCertificateStatus(core.SerialToString(parsedCert.SerialNumber))
+	test.AssertNotError(t, err, "Couldn't get the core.Certificate from the database")
+
+	meta, err := updater.generateResponse(status)
+	test.AssertNotError(t, err, "Couldn't generate OCSP response")
+	tx, err := dbMap.Begin()
+	test.AssertNotError(t, err, "Couldn't open a transaction")
+	err = updater.storeResponse(tx, meta)
+	test.AssertNotError(t, err, "Couldn't store OCSP response")
+	err = tx.Commit()
+	test.AssertNotError(t, err, "Couldn't close transaction")
+
+	certs, err = updater.findStaleOCSPResponses(earliest, 10)
+	test.AssertNotError(t, err, "Failed to find stale responses")
+	test.AssertEquals(t, len(certs), 0)
+}
+
+func TestGetCertificatesWithMissingResponses(t *testing.T) {
+	updater, sa, _, _, cleanUp := setup(t)
+	defer cleanUp()
+
+	reg := satest.CreateWorkingRegistration(t, sa)
+	cert, err := core.LoadCert("test-cert.pem")
+	test.AssertNotError(t, err, "Couldn't read test certificate")
+	_, err = sa.AddCertificate(cert.Raw, reg.ID)
+	test.AssertNotError(t, err, "Couldn't add www.eff.org.der")
+
+	statuses, err := updater.getCertificatesWithMissingResponses(10)
+	test.AssertNotError(t, err, "Couldn't get status")
+	test.AssertEquals(t, len(statuses), 1)
+}
+
+func TestNewCertificateTick(t *testing.T) {
+	updater, sa, _, fc, cleanUp := setup(t)
+	defer cleanUp()
+
+	reg := satest.CreateWorkingRegistration(t, sa)
+	parsedCert, err := core.LoadCert("test-cert.pem")
+	test.AssertNotError(t, err, "Couldn't read test certificate")
+	_, err = sa.AddCertificate(parsedCert.Raw, reg.ID)
+	test.AssertNotError(t, err, "Couldn't add www.eff.org.der")
+
+	prev := fc.Now().Add(-time.Hour)
+	updater.newCertificateTick(10)
+
+	certs, err := updater.findStaleOCSPResponses(prev, 10)
+	test.AssertNotError(t, err, "Failed to find stale responses")
+	test.AssertEquals(t, len(certs), 0)
+}
+
+func TestOldOCSPResponsesTick(t *testing.T) {
+	updater, sa, _, fc, cleanUp := setup(t)
+	defer cleanUp()
+
+	reg := satest.CreateWorkingRegistration(t, sa)
+	parsedCert, err := core.LoadCert("test-cert.pem")
+	test.AssertNotError(t, err, "Couldn't read test certificate")
+	_, err = sa.AddCertificate(parsedCert.Raw, reg.ID)
+	test.AssertNotError(t, err, "Couldn't add www.eff.org.der")
+
+	updater.ocspMinTimeToExpiry = 1 * time.Hour
+	updater.oldOCSPResponsesTick(10)
+
+	certs, err := updater.findStaleOCSPResponses(fc.Now().Add(-updater.ocspMinTimeToExpiry), 10)
+	test.AssertNotError(t, err, "Failed to find stale responses")
+	test.AssertEquals(t, len(certs), 0)
+}
--- a/cmd/ocsp-updater/test-cert-b.pem
+++ b/cmd/ocsp-updater/test-cert-b.pem
@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC6DCCAdCgAwIBAgICALIwDQYJKoZIhvcNAQELBQAwDjEMMAoGA1UEAwwDMTc4
+MB4XDTE1MDYxMzAwMTY1OFoXDTE2MDYxMjAwMTY1OFowDjEMMAoGA1UEAwwDMTc4
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmqs7nue5oFxKBk2WaFZJ
+Ama2nm1oFyPIq19gYEAdQN4mWvaJ8RjzHFkDMYUrlIrGxCYuFJDHFUk9dh19Na1M
+IY+NVLgcSbyNcOML3bLbLEwGmvXPbbEOflBA9mxUS9TLMgXW5ghf/qbt4vmSGKlo
+Iim41QXt55QFW6O+84s8Kd2OE6df0wTsEwLhZB3j5pDU+t7j5vTMv4Tc7EptaPkO
+dfQn+68viUJjlYM/4yIBVRhWCdexFdylCKVLg0obsghQEwULKYCUjdg6F0VJUI11
+5DU49tzscXU/3FS3CyY8rchunuYszBNkdmgpAwViHNWuP7ESdEd/emrj1xuioSe6
+PwIDAQABo1AwTjAdBgNVHQ4EFgQUEaQm2CFKX3v9tNF3LLRNqd5mGcMwHwYDVR0j
+BBgwFoAUEaQm2CFKX3v9tNF3LLRNqd5mGcMwDAYDVR0TBAUwAwEB/zANBgkqhkiG
+9w0BAQsFAAOCAQEAdTi8Mt6JwfXPJU6ILNIXlySl01s7pfNf8Qz43k7AaZSJeI2A
+blM6ilFwbXpWls64XKFQRYfsQ9+wPA044pF1zR05PSI8PJwzIVAjW34myJnbsywb
+Yc1eQXlz0Di7R+w9HRkpVHG2CgnIBGJFa1H7p0FG9tyI7SaJ/Qri5BRJhnu2gYjx
+B+JV3ol+0oYYMhVVaGXwHpyjelsEiWaIFoO3o0YxfW19NM90QQnJ3BGX7ibJSxAr
+Lwbh8DWnWi4X3MdIPG88BKoavcXlJ/pyW2PvarUe31xVBNbyDlcZvrTZ8PXVw7TA
+lumboAhMDLhYNBWrJTJe5LOiapEJaOBNN/ZMFQ==
+-----END CERTIFICATE-----
--- a/cmd/ocsp-updater/test-cert.pem
+++ b/cmd/ocsp-updater/test-cert.pem
@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC6DCCAdCgAwIBAgICAO4wDQYJKoZIhvcNAQELBQAwDjEMMAoGA1UEAwwDMjM4
+MB4XDTE1MDYxMzAwMTU1NVoXDTE2MDYxMjAwMTU1NVowDjEMMAoGA1UEAwwDMjM4
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvP9z1YFDa1WD9hVI9W3K
+lWQmUGfLW35x6xkgDm8o3OTWR2QoxjXratacKhm2VevV22QjCBvHXeHx3fxSp5w/
+p4CH+Ul76wCq3+WAPidO42YCP7SZdqYUR4GHKQ/oOyistRAKEamg4aPAbIs7l1Kn
+T5YHFdSzCWpe6F2+ceoluvKEn6vFVloXKghaeEyTDKnnJKs3/04TdtZjVM5OObvQ
+CGFlQlysDJxWahtVM93gylB8WYgyiekDAx1I3lCd3Vv0hF+x04xT3fwVRzmaKNzT
+wN+znae643Qfg2oSSLV066K2WYepgzqKwv3IUdrLbes331AMs+FbdxHanMrOU1i+
+OQIDAQABo1AwTjAdBgNVHQ4EFgQUjog7s8eJhAvSKMvu6xHZxPnnjsgwHwYDVR0j
+BBgwFoAUjog7s8eJhAvSKMvu6xHZxPnnjsgwDAYDVR0TBAUwAwEB/zANBgkqhkiG
+9w0BAQsFAAOCAQEAIugSn0o0HQMLy02inFZDso4PiRfoqahsT60oIMmWhF3nY3Jq
+GZmkozKGnvyNDeKPlf6TV04VLq6dRg7+yQDL6LCiq2wcGZ+8feMLjyRFwZDSjAJe
+sAMhNq9OQdGNfUV1iZF0SUzqrT68BCT0JTtuDpwlMcmH1O+jFf2HCzROLLBdRC3w
+tJGiA6DH2TqVnucql6sMrnxPVEB+uVfFaKNc9YzwDCp8dSmBbCz7wRmLobGKcnbQ
+lByD5j4dxYkFvJ6n/YX1HKJzwqTWhLQaxvFW7YvnPWepEiXiB6BaIsRgyK7Qa8EW
+3jL5yiB1Dd8OQ7aV7+PNwBNXHd3J1Vie2k52KA==
+-----END CERTIFICATE-----
--- a/cmd/shell.go
+++ b/cmd/shell.go
@ -174,14 +174,7 @@ type Config struct {
 		DebugAddr string
 	}

-	OCSPUpdater struct {
-		DBConnect       string
-		MinTimeToExpiry string
-		ResponseLimit   int
-
-		// DebugAddr is the address to run the /debug handlers on.
-		DebugAddr string
-	}
+	OCSPUpdater OCSPUpdaterConfig

 	Publisher struct {
 		CT publisher.CTConfig
@ -270,6 +263,23 @@ type Queue struct {
 	Server string
 }

+// OCSPUpdaterConfig provides the various window tick times and batch sizes needed
+// for the OCSP (and SCT) updater
+type OCSPUpdaterConfig struct {
+	DBConnect string
+
+	NewCertificateWindow ConfigDuration
+	OldOCSPWindow        ConfigDuration
+
+	NewCertificateBatchSize int
+	OldOCSPBatchSize        int
+
+	OCSPMinTimeToExpiry ConfigDuration
+
+	// DebugAddr is the address to run the /debug handlers on.
+	DebugAddr string
+}
+
 // RateLimitConfig contains all application layer rate limiting policies
 type RateLimitConfig struct {
 	TotalCertificates RateLimitPolicy `yaml:"totalCertificates"`
--- a/core/util.go
+++ b/core/util.go
@ -81,6 +81,9 @@ type SignatureValidationError string
 // for some reason.
 type CertificateIssuanceError string

+// NoSuchRegistrationError indicates that a registration could not be found.
+type NoSuchRegistrationError string
+
 // RateLimitedError indicates the user has hit a rate limit
 type RateLimitedError string

@ -93,6 +96,7 @@ func (e LengthRequiredError) Error() string      { return string(e) }
 func (e SyntaxError) Error() string              { return string(e) }
 func (e SignatureValidationError) Error() string { return string(e) }
 func (e CertificateIssuanceError) Error() string { return string(e) }
+func (e NoSuchRegistrationError) Error() string  { return string(e) }
 func (e RateLimitedError) Error() string         { return string(e) }

 // Base64 functions
--- a/metrics/metrics.go
+++ b/metrics/metrics.go
@ -7,13 +7,13 @@ package metrics

 import (
 	"fmt"
-	"net"
 	"net/http"
 	"strings"
 	"sync/atomic"
 	"time"

 	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
+	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
 )

 // HTTPMonitor stores some server state
@ -22,7 +22,6 @@ type HTTPMonitor struct {
 	statsPrefix         string
 	handler             http.Handler
 	connectionsInFlight int64
-	openConnections     int64
 }

 // NewHTTPMonitor returns a new initialized HTTPMonitor
@ -32,26 +31,9 @@ func NewHTTPMonitor(stats statsd.Statter, handler http.Handler, prefix string) H
 		handler:             handler,
 		statsPrefix:         prefix,
 		connectionsInFlight: 0,
-		openConnections:     0,
 	}
 }

-// ConnectionMonitor provides states on open connection state
-func (h *HTTPMonitor) ConnectionMonitor(_ net.Conn, state http.ConnState) {
-	var open int64
-	switch state {
-	case http.StateNew:
-		open = atomic.AddInt64(&h.openConnections, 1)
-	case http.StateHijacked:
-		fallthrough
-	case http.StateClosed:
-		open = atomic.AddInt64(&h.openConnections, -1)
-	default:
-		return
-	}
-	h.stats.Gauge(fmt.Sprintf("%s.HTTP.OpenConnections", h.statsPrefix), open, 1.0)
-}
-
 // Handle wraps handlers and records various metrics about requests to these handlers
 // and sends them to StatsD
 func (h *HTTPMonitor) Handle() http.Handler {
@ -85,3 +67,53 @@ func (h *HTTPMonitor) watchAndServe(w http.ResponseWriter, r *http.Request) {
 	}
 	h.stats.TimingDuration(fmt.Sprintf("%s.HTTP.ResponseTime.%s", h.statsPrefix, endpoint), cClosed, 1.0)
 }
+
+// FBAdapter provides a facebookgo/stats client interface that sends metrics via
+// a StatsD client
+type FBAdapter struct {
+	stats  statsd.Statter
+	prefix string
+	clk    clock.Clock
+}
+
+// NewFBAdapter returns a new adapter
+func NewFBAdapter(stats statsd.Statter, prefix string, clock clock.Clock) FBAdapter {
+	return FBAdapter{stats: stats, prefix: prefix, clk: clock}
+}
+
+// BumpAvg is essentially statsd.Statter.Gauge
+func (fba FBAdapter) BumpAvg(key string, val float64) {
+	fba.stats.Gauge(fmt.Sprintf("%s.%s", fba.prefix, key), int64(val), 1.0)
+}
+
+// BumpSum is essentially statsd.Statter.Inc (httpdown only ever uses positive
+// deltas)
+func (fba FBAdapter) BumpSum(key string, val float64) {
+	fba.stats.Inc(fmt.Sprintf("%s.%s", fba.prefix, key), int64(val), 1.0)
+}
+
+type btHolder struct {
+	key     string
+	stats   statsd.Statter
+	started time.Time
+}
+
+func (bth btHolder) End() {
+	bth.stats.TimingDuration(bth.key, time.Since(bth.started), 1.0)
+}
+
+// BumpTime is essentially a (much better) statsd.Statter.TimingDuration
+func (fba FBAdapter) BumpTime(key string) interface {
+	End()
+} {
+	return btHolder{
+		key:     fmt.Sprintf("%s.%s", fba.prefix, key),
+		started: fba.clk.Now(),
+		stats:   fba.stats,
+	}
+}
+
+// BumpHistogram isn't used by facebookgo/httpdown
+func (fba FBAdapter) BumpHistogram(_ string, _ float64) {
+	return
+}
--- a/mocks/mocks.go
+++ b/mocks/mocks.go
@ -6,7 +6,6 @@
 package mocks

 import (
-	"database/sql"
 	"encoding/pem"
 	"errors"
 	"fmt"
@ -180,7 +179,7 @@ func (sa *MockSA) GetRegistrationByKey(jwk jose.JsonWebKey) (core.Registration,

 	if core.KeyDigestEquals(jwk, test2KeyPublic) {
 		// No key found
-		return core.Registration{ID: 2}, sql.ErrNoRows
+		return core.Registration{ID: 2}, core.NoSuchRegistrationError("reg not found")
 	}

 	// Return a fake registration. Make sure to fill the key field to avoid marshaling errors.
--- a/ra/registration-authority_test.go
+++ b/ra/registration-authority_test.go
@ -192,10 +192,10 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, *sa.SQLStorageAut
 		OCSPSigner:     ocspSigner,
 		SA:             ssa,
 		PA:             pa,
-		Publisher:      &mocks.MockPublisher{},
 		ValidityPeriod: time.Hour * 2190,
 		NotAfter:       time.Now().Add(time.Hour * 8761),
 		Clk:            fc,
+		Publisher:      &mocks.MockPublisher{},
 	}
 	cleanUp := func() {
 		saDBCleanUp()
--- a/rpc/amqp-rpc.go
+++ b/rpc/amqp-rpc.go
@ -224,6 +224,8 @@ func wrapError(err error) (rpcError RPCError) {
 			rpcError.Type = "SignatureValidationError"
 		case core.CertificateIssuanceError:
 			rpcError.Type = "CertificateIssuanceError"
+		case core.NoSuchRegistrationError:
+			rpcError.Type = "NoSuchRegistrationError"
 		}
 	}
 	return
@ -249,6 +251,8 @@ func unwrapError(rpcError RPCError) (err error) {
 			err = core.SignatureValidationError(rpcError.Value)
 		case "CertificateIssuanceError":
 			err = core.CertificateIssuanceError(rpcError.Value)
+		case "NoSuchRegistrationError":
+			err = core.NoSuchRegistrationError(rpcError.Value)
 		default:
 			err = errors.New(rpcError.Value)
 		}
--- a/sa/_db/migrations/20150929135041_AddCertificateStatusLastOCSPUpdatedIndex.sql
+++ b/sa/_db/migrations/20150929135041_AddCertificateStatusLastOCSPUpdatedIndex.sql
@ -0,0 +1,10 @@
+
+-- +goose Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+CREATE INDEX `ocspLastUpdated_certificateStatus_idx` on `certificateStatus` (`ocspLastUpdated`);
+
+-- +goose Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+DROP INDEX `ocspLastUpdated_certificateStatus_idx` on `certificateStatus`;
--- a/sa/storage-authority.go
+++ b/sa/storage-authority.go
@ -119,14 +119,6 @@ func updateChallenges(authID string, challenges []core.Challenge, tx *gorp.Trans
 	return nil
 }

-type NoSuchRegistrationError struct {
-	Msg string
-}
-
-func (e NoSuchRegistrationError) Error() string {
-	return e.Msg
-}
-
 // GetRegistration obtains a Registration by ID
 func (ssa *SQLStorageAuthority) GetRegistration(id int64) (core.Registration, error) {
 	regObj, err := ssa.dbMap.Get(regModel{}, id)
@ -135,7 +127,7 @@ func (ssa *SQLStorageAuthority) GetRegistration(id int64) (core.Registration, er
 	}
 	if regObj == nil {
 		msg := fmt.Sprintf("No registrations with ID %d", id)
-		return core.Registration{}, NoSuchRegistrationError{Msg: msg}
+		return core.Registration{}, core.NoSuchRegistrationError(msg)
 	}
 	regPtr, ok := regObj.(*regModel)
 	if !ok {
@ -155,7 +147,7 @@ func (ssa *SQLStorageAuthority) GetRegistrationByKey(key jose.JsonWebKey) (core.

 	if err == sql.ErrNoRows {
 		msg := fmt.Sprintf("No registrations with public key sha256 %s", sha)
-		return core.Registration{}, NoSuchRegistrationError{Msg: msg}
+		return core.Registration{}, core.NoSuchRegistrationError(msg)
 	}
 	if err != nil {
 		return core.Registration{}, err
@ -442,7 +434,7 @@ func (ssa *SQLStorageAuthority) UpdateRegistration(reg core.Registration) error
 	}
 	if n == 0 {
 		msg := fmt.Sprintf("Requested registration not found %v", reg.ID)
-		return NoSuchRegistrationError{Msg: msg}
+		return core.NoSuchRegistrationError(msg)
 	}

 	return nil
--- a/sa/storage-authority_test.go
+++ b/sa/storage-authority_test.go
@ -116,18 +116,18 @@ func TestNoSuchRegistrationErrors(t *testing.T) {
 	defer cleanUp()

 	_, err := sa.GetRegistration(100)
-	if _, ok := err.(NoSuchRegistrationError); !ok {
+	if _, ok := err.(core.NoSuchRegistrationError); !ok {
 		t.Errorf("GetRegistration: expected NoSuchRegistrationError, got %T type error (%s)", err, err)
 	}

 	jwk := satest.GoodJWK()
 	_, err = sa.GetRegistrationByKey(jwk)
-	if _, ok := err.(NoSuchRegistrationError); !ok {
+	if _, ok := err.(core.NoSuchRegistrationError); !ok {
 		t.Errorf("GetRegistrationByKey: expected a NoSuchRegistrationError, got %T type error (%s)", err, err)
 	}

 	err = sa.UpdateRegistration(core.Registration{ID: 100, Key: jwk})
-	if _, ok := err.(NoSuchRegistrationError); !ok {
+	if _, ok := err.(core.NoSuchRegistrationError); !ok {
 		t.Errorf("UpdateRegistration: expected a NoSuchRegistrationError, got %T type error (%v)", err, err)
 	}
 }
--- a/test.sh
+++ b/test.sh
@ -5,6 +5,11 @@ if type realpath >/dev/null 2>&1 ; then
  cd $(realpath $(dirname $0))
 fi

+# The list of segments to run. To run only some of these segments, pre-set the
+# RUN variable with the ones you want (see .travis.yml for an example).
+# Order doesn't matter.
+RUN=${RUN:-vet lint fmt migrations unit integration}
+
 FAILURE=0

 TESTPATHS=$(go list -f '{{ .ImportPath }}' ./...)
@ -19,7 +24,7 @@ if [ "x${TRAVIS_PULL_REQUEST}" != "x" ] ; then
  TRIGGER_COMMIT=${revs##* }
 fi

-GITHUB_SECRET_FILE="$(pwd)/test/github-secret.json"
+GITHUB_SECRET_FILE="/tmp/github-secret.json"

 start_context() {
  CONTEXT="$1"
@ -139,116 +144,116 @@ GOBIN=${GOBIN:-$HOME/gopath/bin}
 #
 # Run Go Vet, a correctness-focused static analysis tool
 #
-
-start_context "test/vet"
-run_and_comment go vet ./...
-end_context #test/vet
+if [[ "$RUN" =~ "vet" ]] ; then
+  start_context "test/vet"
+  run_and_comment go vet ./...
+  end_context #test/vet
+fi

 #
 # Run Go Lint, a style-focused static analysis tool
 #
-
-start_context "test/golint"
-[ -x "$(which golint)" ] && run golint ./...
-end_context #test/golint
+if [[ "$RUN" =~ "lint" ]] ; then
+  start_context "test/golint"
+  [ -x "$(which golint)" ] && run golint ./...
+  end_context #test/golint
+fi

 #
 # Ensure all files are formatted per the `go fmt` tool
 #
-start_context "test/gofmt"
-check_gofmt() {
-  unformatted=$(find . -name "*.go" -not -path "./Godeps/*" -print | xargs -n1 gofmt -l)
-  if [ "x${unformatted}" == "x" ] ; then
-    return 0
-  else
-    V="Unformatted files found.
-    Please run 'go fmt' on each of these files and amend your commit to continue."
+if [[ "$RUN" =~ "fmt" ]] ; then
+  start_context "test/gofmt"
+  check_gofmt() {
+    unformatted=$(find . -name "*.go" -not -path "./Godeps/*" -print | xargs -n1 gofmt -l)
+    if [ "x${unformatted}" == "x" ] ; then
+      return 0
+    else
+      V="Unformatted files found.
+      Please run 'go fmt' on each of these files and amend your commit to continue."

-    for f in ${unformatted}; do
-      V=$(printf "%s\n - %s" "${V}" "${f}")
-    done
+      for f in ${unformatted}; do
+        V=$(printf "%s\n - %s" "${V}" "${f}")
+      done

-    # Print to stdout
-    printf "%s\n\n" "${V}"
-    [ "${TRAVIS}" == "true" ] || exit 1 # Stop here if running locally
-    return 1
-  fi
-}
+      # Print to stdout
+      printf "%s\n\n" "${V}"
+      [ "${TRAVIS}" == "true" ] || exit 1 # Stop here if running locally
+      return 1
+    fi
+  }

-run_and_comment check_gofmt
-end_context #test/gofmt
+  run_and_comment check_gofmt
+  end_context #test/gofmt
+fi

-start_context "test/migrations"
-run_and_comment ./test/test-no-outdated-migrations.sh
-end_context "test/migrations"
+if [[ "$RUN" =~ "migrations" ]] ; then
+  start_context "test/migrations"
+  run_and_comment ./test/test-no-outdated-migrations.sh
+  end_context "test/migrations"
+fi

-if [ "${TRAVIS}" == "true" ]; then
+#
+# Prepare the database for unittests and integration tests
+#
+if [[ "${TRAVIS}" == "true" ]] ; then
  ./test/create_db.sh || die "unable to create the boulder database with test/create_db.sh"
 fi

 #
-# Unit Tests. These do not receive a context or status updates,
-# as they are reflected in our eventual exit code.
+# Unit Tests.
 #
-
-if  [ "${SKIP_UNIT_TESTS}" == "1" ]; then
-  echo "Skipping unit tests."
-else
+if [[ "$RUN" =~ "unit" ]] ; then
  run_unit_tests
-fi
-
-# If the unittests failed, exit before trying to run the integration test.
-if [ ${FAILURE} != 0 ]; then
-  echo "--------------------------------------------------"
-  echo "---        A unit test or tool failed.         ---"
-  echo "--- Stopping before running integration tests. ---"
-  echo "--------------------------------------------------"
-  exit ${FAILURE}
-fi
-
-if [ "${SKIP_INTEGRATION_TESTS}" = "1" ]; then
-  echo "Skipping integration tests."
-  exit ${FAILURE}
+  # If the unittests failed, exit before trying to run the integration test.
+  if [ ${FAILURE} != 0 ]; then
+    echo "--------------------------------------------------"
+    echo "---        A unit test or tool failed.         ---"
+    echo "--- Stopping before running integration tests. ---"
+    echo "--------------------------------------------------"
+    exit ${FAILURE}
+  fi
 fi

 #
 # Integration tests
 #
+if [[ "$RUN" =~ "integration" ]] ; then
+  # Set context to integration, and force a pending state
+  start_context "test/integration"
+  update_status --state pending --description "Integration Tests in progress"

-# Set context to integration, and force a pending state
-start_context "test/integration"
-update_status --state pending --description "Integration Tests in progress"
+  if [ -z "$LETSENCRYPT_PATH" ]; then
+    export LETSENCRYPT_PATH=$(mktemp -d -t leXXXX)
+    echo "------------------------------------------------"
+    echo "--- Checking out letsencrypt client is slow. ---"
+    echo "--- Recommend setting \$LETSENCRYPT_PATH to  ---"
+    echo "--- client repo with initialized virtualenv  ---"
+    echo "------------------------------------------------"
+    build_letsencrypt
+  elif [ ! -d "${LETSENCRYPT_PATH}" ]; then
+    build_letsencrypt
+  fi

-if [ -z "$LETSENCRYPT_PATH" ]; then
-  export LETSENCRYPT_PATH=$(mktemp -d -t leXXXX)
-  echo "------------------------------------------------"
-  echo "--- Checking out letsencrypt client is slow. ---"
-  echo "--- Recommend setting \$LETSENCRYPT_PATH to  ---"
-  echo "--- client repo with initialized virtualenv  ---"
-  echo "------------------------------------------------"
-  build_letsencrypt
-elif [ ! -d "${LETSENCRYPT_PATH}" ]; then
-  build_letsencrypt
+  python test/amqp-integration-test.py
+  case $? in
+    0) # Success
+      update_status --state success
+      ;;
+    1) # Python client failed
+      update_status --state success --description "Python integration failed."
+      FAILURE=1
+      ;;
+    2) # Node client failed
+      update_status --state failure --description "NodeJS integration failed."
+      FAILURE=1
+      ;;
+    *) # Error occurred
+      update_status --state error --description "Unknown error occurred."
+      FAILURE=1
+      ;;
+  esac
+  end_context #test/integration
 fi

-python test/amqp-integration-test.py
-case $? in
-  0) # Success
-    update_status --state success
-    ;;
-  1) # Python client failed
-    update_status --state success --description "Python integration failed."
-    FAILURE=1
-    ;;
-  2) # Node client failed
-    update_status --state failure --description "NodeJS integration failed."
-    FAILURE=1
-    ;;
-  *) # Error occurred
-    update_status --state error --description "Unknown error occurred."
-    FAILURE=1
-    ;;
-esac
-end_context #test/integration
-
 exit ${FAILURE}
--- a/test/amqp-integration-test.py
+++ b/test/amqp-integration-test.py
@ -9,6 +9,7 @@ import subprocess
 import sys
 import tempfile
 import urllib
+import time
 import urllib2

 import startservers
@ -84,8 +85,11 @@ def get_ocsp(cert_file, url):
 def verify_ocsp_good(certFile, url):
    output = get_ocsp(certFile, url)
    if not re.search(": good", output):
-        print "Expected OCSP response 'good', got something else."
-        die(ExitStatus.OCSPFailure)
+        if not re.search(" unauthorized \(6\)", output):
+            print "Expected OCSP response 'unauthorized', got something else."
+            die(ExitStatus.OCSPFailure)
+        return False
+    return True

 def verify_ocsp_revoked(certFile, url):
    output = get_ocsp(certFile, url)
@ -94,6 +98,17 @@ def verify_ocsp_revoked(certFile, url):
        die(ExitStatus.OCSPFailure)
    pass

+# loop_check expects the function passed as action will return True/False to indicate
+# success/failure
+def loop_check(failureStatus, action, *args):
+    timeout = time.time() + 5
+    while True:
+        if action(*args):
+            break
+        if time.time() > timeout:
+            die(failureStatus)
+        time.sleep(0.25)
+
 def verify_ct_submission(expectedSubmissions, url):
    resp = urllib2.urlopen(url)
    submissionStr = resp.read()
@ -126,10 +141,15 @@ def run_node_test():

    ee_ocsp_url = "http://localhost:4002"
    issuer_ocsp_url = "http://localhost:4003"
-    verify_ocsp_good(certFile, ee_ocsp_url)
+
    # Also verify that the static OCSP responder, which answers with a
    # pre-signed, long-lived response for the CA cert, also works.
    verify_ocsp_good("../test-ca.der", issuer_ocsp_url)
+
+    # As OCSP-Updater is generating responses indepedantly of the CA we sit in a loop
+    # checking OCSP until we either see a good response or we timeout (5s).
+    loop_check(ExitStatus.OCSPFailure, verify_ocsp_good, certFile, ee_ocsp_url)
+
    verify_ct_submission(1, "http://localhost:4500/submissions")

    if subprocess.Popen('''
--- a/test/boulder-config.json
+++ b/test/boulder-config.json
@ -156,7 +156,14 @@

  "ocspUpdater": {
    "dbConnect": "mysql+tcp://boulder@localhost:3306/boulder_sa_integration",
-    "minTimeToExpiry": "72h",
+    "newCertificateWindow": "1s",
+    "oldOCSPWindow": "2s",
+    "missingSCTWindow": "1m",
+    "newCertificateBatchSize": 1000,
+    "oldOCSPBatchSize": 5000,
+    "missingSCTBatchSize": 5000,
+    "ocspMinTimeToExpiry": "72h",
+    "sctOldestIssued": "72h",
    "debugAddr": "localhost:8006"
  },

--- a/test/startservers.py
+++ b/test/startservers.py
@ -32,29 +32,19 @@ if default_config is None:
 processes = []


-def install(progs, race_detection):
-    cmd = "go install"
+def install(race_detection):
+    # Pass empty BUILD_TIME and BUILD_ID flags to avoid constantly invalidating the
+    # build cache with new BUILD_TIMEs, or invalidating it on merges with a new
+    # BUILD_ID.
+    cmd = "make BUILD_TIME='' BUILD_ID=''  "
    if race_detection:
-        cmd = """go install -race"""
+        cmd = cmd + " GO_BUILD_FLAGS=-race"

-    for prog in progs:
-        cmd += " ./" + prog
-    p = subprocess.Popen(cmd, shell=True)
-    out, err = p.communicate()
-    if p.returncode != 0:
-        sys.stderr.write("unable to run go install: %s\n" % cmd)
-        if out:
-            sys.stderr.write("stdout:\n" + out + "\n")
-        if err:
-            sys.stderr.write("stderr: \n" + err + "\n")
-        return False
-    print('installed %s with pid %d' % (cmd, p.pid))
-    return True
+    return subprocess.call(cmd, shell=True) == 0

-def run(path, race_detection, config=default_config):
-    binary = os.path.basename(path)
+def run(binary, race_detection, config=default_config):
    # Note: Must use exec here so that killing this process kills the command.
-    cmd = """GORACE="halt_on_error=1" exec %s --config %s""" % (binary, config)
+    cmd = """GORACE="halt_on_error=1" exec ./bin/%s --config %s""" % (binary, config)
    p = subprocess.Popen(cmd, shell=True)
    p.cmd = cmd
    print('started %s with pid %d' % (p.cmd, p.pid))
@ -72,17 +62,18 @@ def start(race_detection):
    t.daemon = True
    t.start()
    progs = [
-        'cmd/boulder-wfe',
-        'cmd/boulder-ra',
-        'cmd/boulder-sa',
-        'cmd/boulder-ca',
-        'cmd/boulder-va',
-        'cmd/boulder-publisher',
-        'cmd/ocsp-responder',
-        'test/ct-test-srv',
-        'test/dns-test-srv'
+        'boulder-wfe',
+        'boulder-ra',
+        'boulder-sa',
+        'boulder-ca',
+        'boulder-va',
+        'boulder-publisher',
+        'ocsp-updater',
+        'ocsp-responder',
+        'ct-test-srv',
+        'dns-test-srv'
    ]
-    if not install(progs, race_detection):
+    if not install(race_detection):
        return False
    for prog in progs:
        try:
--- a/test/travis-before-install.sh
+++ b/test/travis-before-install.sh
@ -0,0 +1,35 @@
+#!/bin/bash
+set -o xtrace
+
+# Travis does shallow clones, so there is no master branch present.
+# But test-no-outdated-migrations.sh needs to check diffs against master.
+# Fetch just the master branch from origin.
+git fetch origin master
+git branch master FETCH_HEAD
+# Github-PR-Status secret
+if [ -n "$encrypted_53b2630f0fb4_key" ]; then
+  openssl aes-256-cbc \
+    -K $encrypted_53b2630f0fb4_key -iv $encrypted_53b2630f0fb4_iv \
+    -in test/github-secret.json.enc -out /tmp/github-secret.json -d
+fi
+
+travis_retry go get \
+  golang.org/x/tools/cmd/vet \
+  golang.org/x/tools/cmd/cover \
+  github.com/golang/lint/golint \
+  github.com/mattn/goveralls \
+  github.com/modocache/gover \
+  github.com/jcjones/github-pr-status \
+  github.com/letsencrypt/goose/cmd/goose
+
+# Boulder consists of multiple Go packages, which
+# refer to each other by their absolute GitHub path,
+# e.g. github.com/letsencrypt/boulder/analysis. That means, by default, if
+# someone forks the repo, Travis won't pass on their own repo. To fix that,
+# we add a symlink.
+mkdir -p $TRAVIS_BUILD_DIR $GOPATH/src/github.com/letsencrypt
+if [ ! -d $GOPATH/src/github.com/letsencrypt/boulder ] ; then
+  ln -s $TRAVIS_BUILD_DIR $GOPATH/src/github.com/letsencrypt/boulder
+fi
+
+set +o xtrace
--- a/test/travis_make.sh
+++ b/test/travis_make.sh
@ -1,15 +0,0 @@
-#!/bin/bash
-
-# A script to make it easier to parallelize the build by building make
-# conditionally.
-
-set -o errexit
-
-if [ "${TRAVIS}" != "true" ]; then
-  echo "Not to be run outside of TravisCI" > /dev/stderr
-  exit 1
-fi
-
-if [ "${RUN_MAKE}" == "1" ]; then
-  make -j4 # Travis has 2 cores per build instance
-fi
--- a/va/validation-authority.go
+++ b/va/validation-authority.go
@ -333,30 +333,6 @@ func (va *ValidationAuthorityImpl) validateSimpleHTTP(identifier core.AcmeIdenti
 		return challenge, err
 	}

-	contentTypes, ok := httpResponse.Header["Content-Type"]
-	if ok && len(contentTypes) != 1 {
-		challenge.Status = core.StatusInvalid
-		challenge.Error = &core.ProblemDetails{
-			Type:   core.UnauthorizedProblem,
-			Detail: "Multiple Content-Type headers provided",
-		}
-		return challenge, challenge.Error
-	} else if ok && len(contentTypes) == 1 && contentTypes[0] != "application/jose+json" {
-		challenge.Status = core.StatusInvalid
-		challenge.Error = &core.ProblemDetails{
-			Type:   core.UnauthorizedProblem,
-			Detail: "Invalid content type",
-		}
-		return challenge, challenge.Error
-	} else if !ok {
-		challenge.Status = core.StatusInvalid
-		challenge.Error = &core.ProblemDetails{
-			Type:   core.UnauthorizedProblem,
-			Detail: "No Content-Type header provided",
-		}
-		return challenge, challenge.Error
-	}
-
 	// Read body & test
 	body, readErr := ioutil.ReadAll(httpResponse.Body)
 	if readErr != nil {
--- a/va/validation-authority_test.go
+++ b/va/validation-authority_test.go
@ -92,7 +92,6 @@ func simpleSrv(t *testing.T, token string, enableTLS bool) *httptest.Server {
 	currentToken := defaultToken

 	m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/jose+json")
 		if !strings.HasPrefix(r.Host, "localhost:") && r.Host != "other.valid" && r.Host != "other.valid:8080" {
 			t.Errorf("Bad Host header: " + r.Host)
 		}
@ -132,15 +131,6 @@ func simpleSrv(t *testing.T, token string, enableTLS bool) *httptest.Server {
 		} else if strings.HasSuffix(r.URL.Path, pathRedirectPort) {
 			t.Logf("SIMPLESRV: Got a port redirect req\n")
 			http.Redirect(w, r, "http://other.valid:8080/path", 302)
-		} else if strings.HasSuffix(r.URL.Path, "bad-content-type") {
-			w.Header().Set("Content-Type", "application/bad")
-			t.Logf("SIMPLESRV: Got bad content type header req\n")
-		} else if strings.HasSuffix(r.URL.Path, "multi-content-type") {
-			w.Header()["Content-Type"] = []string{"application/jose+json", "application/bad"}
-			t.Logf("SIMPLESRV: Got bad content type header req\n")
-		} else if strings.HasSuffix(r.URL.Path, "no-content-type") {
-			w.Header().Del("Content-Type")
-			t.Logf("SIMPLESRV: Got bad content type header req\n")
 		} else {
 			t.Logf("SIMPLESRV: Got a valid req\n")
 			fmt.Fprint(w, createValidation(currentToken, enableTLS))
@ -314,30 +304,6 @@ func TestSimpleHttp(t *testing.T) {
 	test.AssertNotError(t, err, "Error validating simpleHttp")
 	test.AssertEquals(t, len(log.GetAllMatching(`^\[AUDIT\] `)), 1)

-	log.Clear()
-	chall.Token = "bad-content-type"
-	invalidChall, err = va.validateSimpleHTTP(ident, chall)
-	test.AssertEquals(t, invalidChall.Status, core.StatusInvalid)
-	test.AssertError(t, err, "Error validating simpleHttp")
-	test.AssertEquals(t, invalidChall.Error.Type, core.UnauthorizedProblem)
-	test.AssertEquals(t, len(log.GetAllMatching(`^\[AUDIT\] `)), 1)
-
-	log.Clear()
-	chall.Token = "multi-content-type"
-	invalidChall, err = va.validateSimpleHTTP(ident, chall)
-	test.AssertEquals(t, invalidChall.Status, core.StatusInvalid)
-	test.AssertError(t, err, "Error validating simpleHttp")
-	test.AssertEquals(t, invalidChall.Error.Type, core.UnauthorizedProblem)
-	test.AssertEquals(t, len(log.GetAllMatching(`^\[AUDIT\] `)), 1)
-
-	log.Clear()
-	chall.Token = "no-content-type"
-	invalidChall, err = va.validateSimpleHTTP(ident, chall)
-	test.AssertEquals(t, invalidChall.Status, core.StatusInvalid)
-	test.AssertError(t, err, "Error validating simpleHttp")
-	test.AssertEquals(t, invalidChall.Error.Type, core.UnauthorizedProblem)
-	test.AssertEquals(t, len(log.GetAllMatching(`^\[AUDIT\] `)), 1)
-
 	log.Clear()
 	chall.Token = path404
 	invalidChall, err = va.validateSimpleHTTP(ident, chall)
--- a/wfe/web-front-end.go
+++ b/wfe/web-front-end.go
@ -8,7 +8,6 @@ package wfe
 import (
 	"bytes"
 	"crypto/x509"
-	"database/sql"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@ -302,9 +301,22 @@ const (
 	malformedJWS = "Unable to read/verify body"
 )

+// verifyPOST reads and parses the request body, looks up the Registration
+// corresponding to its JWK, verifies the JWS signature,
+// checks that the resource field is present and correct in the JWS protected
+// header, and returns the JWS payload bytes, the key used to verify, and the
+// corresponding Registration (or error).
+// If regCheck is false, verifyPOST will still try to look up a registration
+// object, and will return it if found. However, if no registration object is
+// found, verifyPOST will attempt to verify the JWS using the key in the JWS
+// headers, and return the key plus a dummy registration if successful. If a
+// caller passes regCheck = false, it should plan on validating the key itself.
 func (wfe *WebFrontEndImpl) verifyPOST(request *http.Request, regCheck bool, resource core.AcmeResource) ([]byte, *jose.JsonWebKey, core.Registration, error) {
 	var err error
-	var reg core.Registration
+	// TODO: We should return a pointer to a registration, which can be nil,
+	// rather the a registration value with a sentinel value.
+	// https://github.com/letsencrypt/boulder/issues/877
+	reg := core.Registration{ID: -1}

 	if _, ok := request.Header["Content-Length"]; !ok {
 		err = core.LengthRequiredError("Content-Length header is required for POST.")
@ -351,7 +363,34 @@ func (wfe *WebFrontEndImpl) verifyPOST(request *http.Request, regCheck bool, res
 		wfe.log.Debug(err.Error())
 		return nil, nil, reg, err
 	}
-	key := parsedJws.Signatures[0].Header.JsonWebKey
+	submittedKey := parsedJws.Signatures[0].Header.JsonWebKey
+	if submittedKey == nil {
+		err = core.SignatureValidationError("No JWK in JWS header")
+		wfe.log.Debug(err.Error())
+		return nil, nil, reg, err
+	}
+
+	var key *jose.JsonWebKey
+	reg, err = wfe.SA.GetRegistrationByKey(*submittedKey)
+	// Special case: If no registration was found, but regCheck is false, use an
+	// empty registration and the submitted key. The caller is expected to do some
+	// validation on the returned key.
+	if _, ok := err.(core.NoSuchRegistrationError); ok && !regCheck {
+		// When looking up keys from the registrations DB, we can be confident they
+		// are "good". But when we are verifying against any submitted key, we want
+		// to check its quality before doing the verify.
+		if err = core.GoodKey(submittedKey.Key); err != nil {
+			return nil, nil, reg, err
+		}
+		key = submittedKey
+	} else if err != nil {
+		// For all other errors, or if regCheck is true, return error immediately.
+		return nil, nil, reg, err
+	} else {
+		// If the lookup was successful, use that key.
+		key = &reg.Key
+	}
+
 	payload, header, err := parsedJws.Verify(key)
 	if err != nil {
 		puberr := core.SignatureValidationError("JWS verification error")
@ -359,11 +398,6 @@ func (wfe *WebFrontEndImpl) verifyPOST(request *http.Request, regCheck bool, res
 		wfe.log.Debug(fmt.Sprintf("%v :: %v", puberr.Error(), err.Error()))
 		return nil, nil, reg, puberr
 	}
-	if key == nil {
-		err = core.SignatureValidationError("No JWK in JWS header")
-		wfe.log.Debug(err.Error())
-		return nil, nil, reg, err
-	}

 	// Check that the request has a known anti-replay nonce
 	// i.e., Nonce is in protected header and
@ -377,18 +411,6 @@ func (wfe *WebFrontEndImpl) verifyPOST(request *http.Request, regCheck bool, res
 		return nil, nil, reg, err
 	}

-	reg, err = wfe.SA.GetRegistrationByKey(*key)
-	if err != nil {
-		// If we are requiring a valid registration, any failure to look up the
-		// registration is an overall failure to verify.
-		if regCheck {
-			return nil, nil, reg, err
-		}
-		// Otherwise we just return an empty registration. The caller is expected
-		// to use the returned key instead.
-		reg = core.Registration{}
-	}
-
 	// Check that the "resource" field is present and has the correct value
 	var parsedRequest struct {
 		Resource string `json:"resource"`
@ -543,7 +565,7 @@ func (wfe *WebFrontEndImpl) NewAuthorization(response http.ResponseWriter, reque
 		logEvent.Error = err.Error()
 		respMsg := malformedJWS
 		respCode := statusCodeFromError(err)
-		if err == sql.ErrNoRows {
+		if _, ok := err.(core.NoSuchRegistrationError); ok {
 			respMsg = unknownKey
 			respCode = http.StatusForbidden
 		}
@ -713,7 +735,7 @@ func (wfe *WebFrontEndImpl) NewCertificate(response http.ResponseWriter, request
 		logEvent.Error = err.Error()
 		respMsg := malformedJWS
 		respCode := statusCodeFromError(err)
-		if err == sql.ErrNoRows {
+		if _, ok := err.(core.NoSuchRegistrationError); ok {
 			respMsg = unknownKey
 			respCode = http.StatusForbidden
 		}
@ -911,7 +933,7 @@ func (wfe *WebFrontEndImpl) postChallenge(
 		logEvent.Error = err.Error()
 		respMsg := malformedJWS
 		respCode := http.StatusBadRequest
-		if err == sql.ErrNoRows {
+		if _, ok := err.(core.NoSuchRegistrationError); ok {
 			respMsg = unknownKey
 			respCode = http.StatusForbidden
 		}
@ -987,7 +1009,7 @@ func (wfe *WebFrontEndImpl) Registration(response http.ResponseWriter, request *
 		logEvent.Error = err.Error()
 		respMsg := malformedJWS
 		respCode := statusCodeFromError(err)
-		if err == sql.ErrNoRows {
+		if _, ok := err.(core.NoSuchRegistrationError); ok {
 			respMsg = unknownKey
 			respCode = http.StatusForbidden
 		}
--- a/wfe/web-front-end_test.go
+++ b/wfe/web-front-end_test.go
@ -184,11 +184,10 @@ func makeBody(s string) io.ReadCloser {
 }

 func signRequest(t *testing.T, req string, nonceService *core.NonceService) string {
-	accountKeyJSON := []byte(`{"kty":"RSA","n":"z2NsNdHeqAiGdPP8KuxfQXat_uatOK9y12SyGpfKw1sfkizBIsNxERjNDke6Wp9MugN9srN3sr2TDkmQ-gK8lfWo0v1uG_QgzJb1vBdf_hH7aejgETRGLNJZOdaKDsyFnWq1WGJq36zsHcd0qhggTk6zVwqczSxdiWIAZzEakIUZ13KxXvoepYLY0Q-rEEQiuX71e4hvhfeJ4l7m_B-awn22UUVvo3kCqmaRlZT-36vmQhDGoBsoUo1KBEU44jfeK5PbNRk7vDJuH0B7qinr_jczHcvyD-2TtPzKaCioMtNh_VZbPNDaG67sYkQlC15-Ff3HPzKKJW2XvkVG91qMvQ","e":"AQAB","d":"BhAmDbzBAbCeHbU0Xhzi_Ar4M0eTMOEQPnPXMSfW6bc0SRW938JO_-z1scEvFY8qsxV_C0Zr7XHVZsmHz4dc9BVmhiSan36XpuOS85jLWaY073e7dUVN9-l-ak53Ys9f6KZB_v-BmGB51rUKGB70ctWiMJ1C0EzHv0h6Moog-LCd_zo03uuZD5F5wtnPrAB3SEM3vRKeZHzm5eiGxNUsaCEzGDApMYgt6YkQuUlkJwD8Ky2CkAE6lLQSPwddAfPDhsCug-12SkSIKw1EepSHz86ZVfJEnvY-h9jHIdI57mR1v7NTCDcWqy6c6qIzxwh8n2X94QTbtWT3vGQ6HXM5AQ","p":"2uhvZwNS5i-PzeI9vGx89XbdsVmeNjVxjH08V3aRBVY0dzUzwVDYk3z7sqBIj6de53Lx6W1hjmhPIqAwqQgjIKH5Z3uUCinGguKkfGDL3KgLCzYL2UIvZMvTzr9NWLc0AHMZdee5utxWKCGnZBOqy1Rd4V-6QrqjEDBvanoqA60","q":"8odNkMEiriaDKmvwDv-vOOu3LaWbu03yB7VhABu-hK5Xx74bHcvDP2HuCwDGGJY2H-xKdMdUPs0HPwbfHMUicD2vIEUDj6uyrMMZHtbcZ3moh3-WESg3TaEaJ6vhwcWXWG7Wc46G-HbCChkuVenFYYkoi68BAAjloqEUl1JBT1E"}`)
-	var accountKey jose.JsonWebKey
-	err := json.Unmarshal(accountKeyJSON, &accountKey)
-	test.AssertNotError(t, err, "Failed to unmarshal key")
-	signer, err := jose.NewSigner("RS256", &accountKey)
+	accountKey, err := jose.LoadPrivateKey([]byte(test1KeyPrivatePEM))
+	test.AssertNotError(t, err, "Failed to load key")
+
+	signer, err := jose.NewSigner("RS256", accountKey)
 	test.AssertNotError(t, err, "Failed to make signer")
 	nonce, err := nonceService.Nonce()
 	test.AssertNotError(t, err, "Failed to make nonce")
@ -769,6 +768,18 @@ func makeRevokeRequestJSON() ([]byte, error) {
 	return revokeRequestJSON, nil
 }

+// An SA mock that always returns NoSuchRegistrationError. This is necessary
+// because the standard mock in our mocks package always returns a given test
+// registration when GetRegistrationByKey is called, and we want to get a
+// NoSuchRegistrationError for tests that pass regCheck = false to verifyPOST.
+type mockSANoSuchRegistration struct {
+	mocks.MockSA
+}
+
+func (msa mockSANoSuchRegistration) GetRegistrationByKey(jwk jose.JsonWebKey) (core.Registration, error) {
+	return core.Registration{}, core.NoSuchRegistrationError("reg not found")
+}
+
 // Valid revocation request for existing, non-revoked cert, signed with cert
 // key.
 func TestRevokeCertificateCertKey(t *testing.T) {
@ -785,6 +796,7 @@ func TestRevokeCertificateCertKey(t *testing.T) {
 	test.AssertNotError(t, err, "Failed to make revokeRequestJSON")

 	wfe := setupWFE(t)
+	wfe.SA = &mockSANoSuchRegistration{mocks.MockSA{}}
 	responseWriter := httptest.NewRecorder()

 	nonce, err := wfe.nonceService.Nonce()
@ -874,7 +886,7 @@ func TestRevokeCertificateAlreadyRevoked(t *testing.T) {
 	wfe := setupWFE(t)

 	wfe.RA = &MockRegistrationAuthority{}
-	wfe.SA = &mocks.MockSA{}
+	wfe.SA = &mockSANoSuchRegistration{mocks.MockSA{}}
 	wfe.stats, _ = statsd.NewNoopClient()
 	wfe.SubscriberAgreementURL = agreementURL
 	responseWriter := httptest.NewRecorder()
@ -1206,6 +1218,29 @@ func TestLengthRequired(t *testing.T) {
 	test.Assert(t, ok, "Error code for missing content-length wasn't 411.")
 }

+type mockSADifferentStoredKey struct {
+	mocks.MockSA
+}
+
+func (sa mockSADifferentStoredKey) GetRegistrationByKey(jwk jose.JsonWebKey) (core.Registration, error) {
+	keyJSON := []byte(test2KeyPublicJSON)
+	var parsedKey jose.JsonWebKey
+	parsedKey.UnmarshalJSON(keyJSON)
+
+	return core.Registration{
+		Key: parsedKey,
+	}, nil
+}
+
+func TestVerifyPOSTUsesStoredKey(t *testing.T) {
+	wfe := setupWFE(t)
+	wfe.SA = &mockSADifferentStoredKey{mocks.MockSA{}}
+	// signRequest signs with test1Key, but our special mock returns a
+	// registration with test2Key
+	_, _, _, err := wfe.verifyPOST(makePostRequest(signRequest(t, `{"resource":"foo"}`, &wfe.nonceService)), true, "foo")
+	test.AssertError(t, err, "No error returned when provided key differed from stored key.")
+}
+
 func TestBadKeyCSR(t *testing.T) {
 	wfe := setupWFE(t)
 	responseWriter := httptest.NewRecorder()