letsencrypt
/
boulder
mirror of https://github.com/letsencrypt/boulder.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add time-dependent integration testing (#3060) 

Fixes #3020.

In order to write integration tests for some features, especially related to rate limiting, rechecking of CAA, and expiration of authzs, orders, and certs, we need to be able to fake the passage of time in integration tests.

To do so, this change switches out all clock.Default() instances for cmd.Clock(), which can be set manually with the FAKECLOCK environment variable. integration-test.py now starts up all servers once before the main body of tests, with FAKECLOCK set to a date 70 days ago, and does some initial setup for a new integration test case. That test case tries to fetch a 70-day-old authz URL, and expects it to 404.

In order to make this work, I also had to change a number of our test binaries to shut down cleanly in response to SIGTERM. Without that change, stopping the servers between the setup phase and the main tests caused startservers.check() to fail, because some processes exited with nonzero status.

Note: This is an initial stab at things, to prove out the technique. Long-term, I think we will want to use an idiom where test cases are classes that have a number of optional setup phases that may be run at e.g. 70 days prior and 5 days prior. This could help us avoid a proliferation of global state as we add more time-dependent test cases.
This commit is contained in:
Jacob Hoffman-Andrews 2017-09-13 12:34:14 -07:00 committed by Roland Bracewell Shoemaker
parent c03d96212b
commit 4128e0d95a
17 changed files with 83 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/boulder-ca/main.go
View File

@ -11,7 +11,6 @@ import (
"os"
"github.com/cloudflare/cfssl/helpers"
"github.com/jmhodges/clock"
"github.com/letsencrypt/pkcs11key"
"google.golang.org/grpc"
@ -170,7 +169,7 @@ func main() {
c.CA,
sa,
pa,
clock.Default(),
cmd.Clock(),
scope,
issuers,
kp,

7
cmd/boulder-ra/main.go
View File

@ -8,7 +8,6 @@ import (
"os"
"time"
"github.com/jmhodges/clock"
"google.golang.org/grpc"
"github.com/letsencrypt/boulder/bdns"
@ -166,7 +165,7 @@ func main() {
cmd.FailOnError(err, "Unable to create key policy")
rai := ra.NewRegistrationAuthorityImpl(
clock.Default(),
cmd.Clock(),
logger,
scope,
c.RA.MaxContactsPerRegistration,
@ -197,14 +196,14 @@ func main() {
[]string{c.Common.DNSResolver},
nil,
scope,
clock.Default(),
cmd.Clock(),
dnsTries)
} else {
rai.DNSClient = bdns.NewTestDNSClientImpl(
raDNSTimeout,
[]string{c.Common.DNSResolver},
scope,
clock.Default(),
cmd.Clock(),
dnsTries)
}

3
cmd/boulder-sa/main.go
View File

@ -5,7 +5,6 @@ import (
"net"
"os"
"github.com/jmhodges/clock"
"google.golang.org/grpc"
"github.com/letsencrypt/boulder/cmd"
@ -55,7 +54,7 @@ func main() {
go sa.ReportDbConnCount(dbMap, scope)
sai, err := sa.NewSQLStorageAuthority(dbMap, clock.Default(), logger, scope)
sai, err := sa.NewSQLStorageAuthority(dbMap, cmd.Clock(), logger, scope)
cmd.FailOnError(err, "Failed to create SA impl")
var grpcSrv *grpc.Server

4
cmd/boulder-va/main.go
View File

@ -6,8 +6,6 @@ import (
"strings"
"time"
"github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/bdns"
"github.com/letsencrypt/boulder/cmd"
"github.com/letsencrypt/boulder/features"
@ -96,7 +94,7 @@ func main() {
if dnsTries < 1 {
dnsTries = 1
}
clk := clock.Default()
clk := cmd.Clock()
caaSERVFAILExceptions, err := bdns.ReadHostList(c.VA.CAASERVFAILExceptions)
cmd.FailOnError(err, "Couldn't read CAASERVFAILExceptions file")
var resolver bdns.DNSClient

3
cmd/boulder-wfe/main.go
View File

@ -8,7 +8,6 @@ import (
"os"
"github.com/facebookgo/httpdown"
"github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/cmd"
"github.com/letsencrypt/boulder/core"
@ -105,7 +104,7 @@ func main() {
kp, err := goodkey.NewKeyPolicy("") // don't load any weak keys
cmd.FailOnError(err, "Unable to create key policy")
wfe, err := wfe.NewWebFrontEndImpl(scope, clock.Default(), kp, logger)
wfe, err := wfe.NewWebFrontEndImpl(scope, cmd.Clock(), kp, logger)
cmd.FailOnError(err, "Unable to create WFE")
rac, sac := setupWFE(c, logger, scope)
wfe.RA = rac

3
cmd/boulder-wfe2/main.go
View File

@ -8,7 +8,6 @@ import (
"os"
"github.com/facebookgo/httpdown"
"github.com/jmhodges/clock"
"github.com/letsencrypt/boulder/cmd"
"github.com/letsencrypt/boulder/core"
@ -105,7 +104,7 @@ func main() {
kp, err := goodkey.NewKeyPolicy("") // don't load any weak keys
cmd.FailOnError(err, "Unable to create key policy")
wfe, err := wfe2.NewWebFrontEndImpl(scope, clock.Default(), kp, logger)
wfe, err := wfe2.NewWebFrontEndImpl(scope, cmd.Clock(), kp, logger)
cmd.FailOnError(err, "Unable to create WFE")
rac, sac := setupWFE(c, logger, scope)
wfe.RA = rac

2
cmd/cert-checker/main.go
View File

@ -332,7 +332,7 @@ func main() {
checker := newChecker(
saDbMap,
clock.Default(),
cmd.Clock(),
pa,
config.CertChecker.CheckPeriod.Duration,
)

3
cmd/ocsp-responder/main.go
View File

@ -15,7 +15,6 @@ import (
cfocsp "github.com/cloudflare/cfssl/ocsp"
"github.com/facebookgo/httpdown"
"github.com/jmhodges/clock"
"golang.org/x/crypto/ocsp"
"github.com/letsencrypt/boulder/cmd"
@ -254,5 +253,5 @@ func mux(scope metrics.Scope, responderPath string, source cfocsp.Source) http.H
}
stripPrefix.ServeHTTP(w, r)
})
return measured_http.New(&ocspMux{h}, clock.Default())
return measured_http.New(&ocspMux{h}, cmd.Clock())
}

13
cmd/ocsp-updater/main.go
View File

@ -757,9 +757,9 @@ func main() {
err = features.Set(conf.Features)
cmd.FailOnError(err, "Failed to set feature flags")
scope, auditlogger := cmd.StatsAndLogging(c.Syslog)
defer auditlogger.AuditPanic()
auditlogger.Info(cmd.VersionString())
scope, logger := cmd.StatsAndLogging(c.Syslog)
defer logger.AuditPanic()
logger.Info(cmd.VersionString())
// Configure DB
dbURL, err := conf.DBConfig.URL()
@ -772,7 +772,7 @@ func main() {
updater, err := newUpdater(
scope,
clock.Default(),
cmd.Clock(),
dbMap,
cac,
pubc,
@ -781,7 +781,7 @@ func main() {
conf,
c.Common.CT.Logs,
c.Common.IssuerCert,
auditlogger,
logger,
)
cmd.FailOnError(err, "Failed to create updater")
@ -790,11 +790,12 @@ func main() {
go func(loop *looper) {
err = loop.loop()
if err != nil {
auditlogger.AuditErr(err.Error())
logger.AuditErr(err.Error())
}
}(l)
}
go cmd.CatchSignals(logger, nil)
go cmd.DebugServer(conf.DebugAddr)
go cmd.ProfileCmd(scope)

4
cmd/shell.go
View File

@ -272,13 +272,17 @@ func CatchSignals(logger blog.Logger, callback func()) {
signal.Notify(sigChan, syscall.SIGHUP)
sig := <-sigChan
if logger != nil {
logger.Info(fmt.Sprintf("Caught %s", signalToName[sig]))
}
if callback != nil {
callback()
}
if logger != nil {
logger.Info("Exiting")
}
os.Exit(0)
}

2
test/chisel.py
View File

@ -169,7 +169,7 @@ def auth_and_issue(domains, chall_type="http-01", email=None, cert_output=None,
try:
cert_resource = issue(client, authzs, cert_output)
client.fetch_chain(cert_resource)
return cert_resource
return cert_resource, authzs
finally:
cleanup()

3
test/ct-test-srv/main.go
View File

@ -21,6 +21,7 @@ import (
ct "github.com/google/certificate-transparency-go"
ctTLS "github.com/google/certificate-transparency-go/tls"
"github.com/letsencrypt/boulder/cmd"
)
func createSignedSCT(leaf []byte, k *ecdsa.PrivateKey) []byte {
@ -159,5 +160,5 @@ func main() {
go func() { log.Fatal(sA.ListenAndServe()) }()
go func() { log.Fatal(sB.ListenAndServe()) }()
select {}
cmd.CatchSignals(nil, nil)
}

4
test/dns-test-srv/main.go
View File

@ -11,6 +11,7 @@ import (
"sync"
"time"
"github.com/letsencrypt/boulder/cmd"
"github.com/miekg/dns"
)
@ -175,6 +176,5 @@ func (ts *testSrv) serveTestResolver() {
func main() {
ts := testSrv{mu: new(sync.RWMutex), txtRecords: make(map[string]string)}
ts.serveTestResolver()
forever := make(chan bool, 1)
<-forever
cmd.CatchSignals(nil, nil)
}

5
test/gsb-test-srv/main.go
View File

@ -14,6 +14,7 @@ import (
"sync"
"github.com/golang/protobuf/proto"
"github.com/letsencrypt/boulder/cmd"
gsb "github.com/letsencrypt/boulder/test/gsb-test-srv/proto"
)
@ -410,7 +411,5 @@ func main() {
ts := newTestServer(*key, defaultUnsafeURLs)
ts.start(*listen)
// Block on an empty channel
forever := make(chan bool, 1)
<-forever
cmd.CatchSignals(nil, nil)
}

57
test/integration-test.py
View File

@ -8,6 +8,7 @@ import json
import os
import random
import re
import requests
import shutil
import subprocess
import signal
@ -32,6 +33,17 @@ class ProcInfo:
self.cmd = cmd
self.proc = proc
old_authzs = []
def setup_seventy_days_ago():
"""Do any setup that needs to happen 70 days in the past, for tests that
will run in the 'present'.
"""
# Issue a certificate with the clock set back, and save the authzs to check
# later that they are expired (404).
global old_authzs
_, old_authzs = auth_and_issue([random_domain()])
def fetch_ocsp(request_bytes, url):
"""Fetch an OCSP response using POST, GET, and GET with URL encoding.
@ -205,9 +217,9 @@ def random_domain():
def test_expiration_mailer():
email_addr = "integration.%x@boulder.local" % random.randrange(2**16)
cert = auth_and_issue([random_domain()], email=email_addr).body
cert, _ = auth_and_issue([random_domain()], email=email_addr)
# Check that the expiration mailer sends a reminder
expiry = datetime.datetime.strptime(cert.get_notAfter(), '%Y%m%d%H%M%SZ')
expiry = datetime.datetime.strptime(cert.body.get_notAfter(), '%Y%m%d%H%M%SZ')
no_reminder = expiry + datetime.timedelta(days=-31)
first_reminder = expiry + datetime.timedelta(days=-13)
last_reminder = expiry + datetime.timedelta(days=-2)
@ -227,7 +239,7 @@ def test_expiration_mailer():
def test_revoke_by_account():
cert_file_pem = os.path.join(tempdir, "revokeme.pem")
client = chisel.make_client()
cert = auth_and_issue([random_domain()], client=client).body
cert, _ = auth_and_issue([random_domain()], client=client)
client.revoke(cert.body)
wait_for_ocsp_revoked(cert_file_pem, "test/test-ca2.pem", ee_ocsp_url)
@ -277,8 +289,11 @@ def test_single_ocsp():
p.send_signal(signal.SIGTERM)
p.wait()
def fakeclock(date):
return date.strftime("%a %b %d %H:%M:%S UTC %Y")
def get_future_output(cmd, date):
return run(cmd, env={'FAKECLOCK': date.strftime("%a %b %d %H:%M:%S UTC %Y")})
return run(cmd, env={'FAKECLOCK': fakeclock(date)})
def test_expired_authz_purger():
def expect(target_time, num, table):
@ -345,14 +360,23 @@ def test_certificates_per_name():
chisel.expect_problem("urn:acme:error:rateLimited",
lambda: auth_and_issue(["lim.it"]))
def test_expired_authzs_404():
if len(old_authzs) == 0:
raise Exception("Old authzs not prepared for test_expired_authzs_404")
for a in old_authzs:
response = requests.get(a.uri)
if response.status_code != 404:
raise Exception("Unexpected response for expired authz: ",
response.status_code)
default_config_dir = os.environ.get('BOULDER_CONFIG_DIR', '')
if default_config_dir == '':
default_config_dir = 'test/config'
def test_admin_revoker_cert():
cert_file_pem = os.path.join(tempdir, "ar-cert.pem")
cert = auth_and_issue([random_domain()], cert_output=cert_file_pem).body
serial = "%x" % cert.get_serial_number()
cert, _ = auth_and_issue([random_domain()], cert_output=cert_file_pem)
serial = "%x" % cert.body.get_serial_number()
# Revoke certificate by serial
run("./bin/admin-revoker serial-revoke --config %s/admin-revoker.json %s %d" % (
default_config_dir, serial, 1))
@ -395,22 +419,20 @@ def main():
if not (args.run_all or args.run_certbot or args.run_chisel or args.custom is not None):
raise Exception("must run at least one of the letsencrypt or chisel tests with --all, --certbot, --chisel, or --custom")
# Keep track of whether we started the Boulder servers and need to shut them down.
started_servers = False
# Check if WFE is already running.
try:
urllib2.urlopen("http://localhost:4000/directory")
except urllib2.URLError:
# WFE not running, start all of Boulder.
started_servers = True
now = datetime.datetime.utcnow()
seventy_days_ago = now+datetime.timedelta(days=-70)
if not startservers.start(race_detection=True, fakeclock=fakeclock(seventy_days_ago)):
raise Exception("startservers failed (mocking seventy days ago)")
setup_seventy_days_ago()
startservers.stop()
if not startservers.start(race_detection=True):
raise Exception("startservers failed")
if args.run_all or args.run_chisel:
run_chisel()
# Simulate a disconnection from RabbitMQ to make sure reconnects work.
if started_servers:
# Simulate a disconnection to make sure gRPC reconnects work.
startservers.bounce_forward()
if args.run_all or args.run_certbot:
@ -419,7 +441,7 @@ def main():
if args.custom:
run(args.custom)
if started_servers and not startservers.check():
if not startservers.check():
raise Exception("startservers.check failed")
global exit_status
@ -441,6 +463,7 @@ def run_chisel():
test_single_ocsp()
test_dns_challenge()
test_renewal_exemption()
test_expired_authzs_404()
if __name__ == "__main__":
try:

3
test/mail-test-srv/main.go
View File

@ -14,6 +14,7 @@ import (
"sync"
"time"
"github.com/letsencrypt/boulder/cmd"
blog "github.com/letsencrypt/boulder/log"
)
@ -216,6 +217,8 @@ func main() {
}
}()
go cmd.CatchSignals(nil, nil)
err = srv.serveSMTP(l)
if err != nil {
log.Fatalln(err, "Failed to accept connection")

10
test/startservers.py
View File

@ -28,16 +28,18 @@ def install(race_detection):
return subprocess.call(cmd, shell=True) == 0
def run(cmd, race_detection):
def run(cmd, race_detection, fakeclock):
e = os.environ.copy()
e.setdefault("GORACE", "halt_on_error=1")
if fakeclock is not None:
e.setdefault("FAKECLOCK", fakeclock)
# Note: Must use exec here so that killing this process kills the command.
cmd = """exec ./bin/%s""" % cmd
p = subprocess.Popen(cmd, shell=True, env=e)
p.cmd = cmd
return p
def start(race_detection):
def start(race_detection, fakeclock=None):
"""Return True if everything builds and starts.
Give up and return False if anything fails to build, or dies at
@ -85,7 +87,7 @@ def start(race_detection):
return False
for prog in progs:
try:
processes.append(run(prog, race_detection))
processes.append(run(prog, race_detection, fakeclock))
except Exception as e:
print(e)
return False
@ -165,8 +167,10 @@ def stop():
# When we are about to exit, send SIGTERM to each subprocess and wait for
# them to nicely die. This reflects the restart process in prod and allows
# us to exercise the graceful shutdown code paths.
global processes
for p in processes:
if p.poll() is None:
p.send_signal(signal.SIGTERM)
for p in processes:
p.wait()
processes = []