From 3a4fef4463a0080b791a11d0fd5fe34ca70b399c Mon Sep 17 00:00:00 2001 From: Jeff Hodges Date: Tue, 25 Aug 2015 15:59:25 -0700 Subject: [PATCH 1/5] install boulder cmds in one cmd in startserver.py This eases the CPU and thread requirements of our tests (by forking less, not doing everything at once). It should also speed up the tests by avoiding certain repetitive work. Updates https://github.com/letsencrypt/letsencrypt/issues/712 --- test/startservers.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/test/startservers.py b/test/startservers.py index 491eeff25..a10afc0be 100644 --- a/test/startservers.py +++ b/test/startservers.py @@ -32,19 +32,31 @@ if config is None: processes = [] -def run(path, race_detection): +def install(progs, race_detection): install = "go install" if race_detection: - install = """GORACE="halt_on_error=1" go install -race""" + install = """go install -race""" + cmd = install + for prog in progs: + cmd += " ./" + prog + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + if p.returncode != 0: + sys.stderr.write("unable to run go install: %s\n" % cmd) + sys.stderr.write("stdout:\n" + out + "\n") + sys.stderr.write("stderr: \n" + err + "\n") + return False + print('installed %s with pid %d' % (cmd, p.pid)) + return True +def run(path, race_detection): binary = os.path.basename(path) - cmd = """%s ./%s && exec %s --config %s""" % (install, path, binary, config) + cmd = """GORACE="halt_on_error=1" %s --config %s""" % (binary, config) p = subprocess.Popen(cmd, shell=True) p.cmd = cmd print('started %s with pid %d' % (p.cmd, p.pid)) return p - def start(race_detection): """Return True if everything builds and starts. @@ -56,14 +68,18 @@ def start(race_detection): t = ToSServerThread() t.daemon = True t.start() - for prog in [ - 'cmd/boulder-wfe', - 'cmd/boulder-ra', - 'cmd/boulder-sa', - 'cmd/boulder-ca', - 'cmd/boulder-va', - 'cmd/ocsp-responder', - 'test/dns-test-srv']: + progs = [ + 'cmd/boulder-wfe', + 'cmd/boulder-ra', + 'cmd/boulder-sa', + 'cmd/boulder-ca', + 'cmd/boulder-va', + 'cmd/ocsp-responder', + 'test/dns-test-srv' + ] + if not install(progs, race_detection): + return False + for prog in progs: try: processes.append(run(prog, race_detection)) except Exception as e: From 15cb96933f275305ff440fc9103162bc57d0ba6e Mon Sep 17 00:00:00 2001 From: Jeff Hodges Date: Tue, 25 Aug 2015 16:30:58 -0700 Subject: [PATCH 2/5] remove unused private func in ca --- ca/certificate-authority.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ca/certificate-authority.go b/ca/certificate-authority.go index 0b274e6d7..ecb0fce97 100644 --- a/ca/certificate-authority.go +++ b/ca/certificate-authority.go @@ -213,20 +213,6 @@ func loadIssuer(filename string) (issuerCert *x509.Certificate, err error) { return } -func loadIssuerKey(filename string) (issuerKey crypto.Signer, err error) { - if filename == "" { - err = errors.New("IssuerKey must be provided in test mode.") - return - } - - pem, err := ioutil.ReadFile(filename) - if err != nil { - return - } - issuerKey, err = helpers.ParsePrivateKeyPEM(pem) - return -} - // GenerateOCSP produces a new OCSP response and returns it func (ca *CertificateAuthorityImpl) GenerateOCSP(xferObj core.OCSPSigningRequest) ([]byte, error) { cert, err := x509.ParseCertificate(xferObj.CertDER) From 5bd820f3c5c59ff7825c77d2a5b0396aea47e4a3 Mon Sep 17 00:00:00 2001 From: Jeff Hodges Date: Tue, 25 Aug 2015 21:12:07 -0700 Subject: [PATCH 3/5] avoid copying large ValidationAuthorityImpl struct Saves some allocations --- cmd/boulder-va/main.go | 2 +- va/validation-authority.go | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/boulder-va/main.go b/cmd/boulder-va/main.go index a3f4b6e7f..518af3b15 100644 --- a/cmd/boulder-va/main.go +++ b/cmd/boulder-va/main.go @@ -54,7 +54,7 @@ func main() { vas, err := rpc.NewAmqpRPCServer(c.AMQP.VA.Server, connectionHandler) cmd.FailOnError(err, "Unable to create VA RPC server") - rpc.NewValidationAuthorityServer(vas, &vai) + rpc.NewValidationAuthorityServer(vas, vai) auditlogger.Info(app.VersionString()) diff --git a/va/validation-authority.go b/va/validation-authority.go index d08e0b91b..b42dd799c 100644 --- a/va/validation-authority.go +++ b/va/validation-authority.go @@ -52,20 +52,20 @@ type ValidationAuthorityImpl struct { // NewValidationAuthorityImpl constructs a new VA, and may place it // into Test Mode (tm) -func NewValidationAuthorityImpl(tm bool) ValidationAuthorityImpl { +func NewValidationAuthorityImpl(tm bool) *ValidationAuthorityImpl { logger := blog.GetAuditLogger() logger.Notice("Validation Authority Starting") // TODO(jsha): Remove TestMode entirely. Instead, the various validation ports // should be exported, so the cmd file can set them based on a config. if tm { - return ValidationAuthorityImpl{ + return &ValidationAuthorityImpl{ log: logger, simpleHTTPPort: 5001, simpleHTTPSPort: 5001, dvsniPort: 5001, } } else { - return ValidationAuthorityImpl{ + return &ValidationAuthorityImpl{ log: logger, simpleHTTPPort: 80, simpleHTTPSPort: 443, @@ -142,7 +142,7 @@ func problemDetailsFromDNSError(err error) *core.ProblemDetails { // This is the same choice made by the Go internal resolution library used by // net/http, except we only send A queries and accept IPv4 addresses. // TODO(#593): Add IPv6 support -func (va ValidationAuthorityImpl) getAddr(hostname string) (addr net.IP, addrs []net.IP, problem *core.ProblemDetails) { +func (va *ValidationAuthorityImpl) getAddr(hostname string) (addr net.IP, addrs []net.IP, problem *core.ProblemDetails) { addrs, _, err := va.DNSResolver.LookupHost(hostname) if err != nil { problem = problemDetailsFromDNSError(err) @@ -172,7 +172,7 @@ func (d *dialer) Dial(_, _ string) (net.Conn, error) { // resolveAndConstructDialer gets the prefered address using va.getAddr and returns // the chosen address and dialer for that address and correct port. -func (va ValidationAuthorityImpl) resolveAndConstructDialer(name, defaultPort string) (dialer, *core.ProblemDetails) { +func (va *ValidationAuthorityImpl) resolveAndConstructDialer(name, defaultPort string) (dialer, *core.ProblemDetails) { port := fmt.Sprintf("%d", va.simpleHTTPPort) if defaultPort != "" { port = defaultPort @@ -195,7 +195,7 @@ func (va ValidationAuthorityImpl) resolveAndConstructDialer(name, defaultPort st // Validation methods -func (va ValidationAuthorityImpl) validateSimpleHTTP(identifier core.AcmeIdentifier, input core.Challenge, accountKey jose.JsonWebKey) (core.Challenge, error) { +func (va *ValidationAuthorityImpl) validateSimpleHTTP(identifier core.AcmeIdentifier, input core.Challenge, accountKey jose.JsonWebKey) (core.Challenge, error) { challenge := input if identifier.Type != core.IdentifierDNS { @@ -376,7 +376,7 @@ func (va ValidationAuthorityImpl) validateSimpleHTTP(identifier core.AcmeIdentif return challenge, nil } -func (va ValidationAuthorityImpl) validateDvsni(identifier core.AcmeIdentifier, input core.Challenge, accountKey jose.JsonWebKey) (core.Challenge, error) { +func (va *ValidationAuthorityImpl) validateDvsni(identifier core.AcmeIdentifier, input core.Challenge, accountKey jose.JsonWebKey) (core.Challenge, error) { challenge := input if identifier.Type != "dns" { @@ -497,7 +497,7 @@ func parseHTTPConnError(err error) core.ProblemType { return core.ConnectionProblem } -func (va ValidationAuthorityImpl) validateDNS(identifier core.AcmeIdentifier, input core.Challenge, accountKey jose.JsonWebKey) (core.Challenge, error) { +func (va *ValidationAuthorityImpl) validateDNS(identifier core.AcmeIdentifier, input core.Challenge, accountKey jose.JsonWebKey) (core.Challenge, error) { challenge := input if identifier.Type != core.IdentifierDNS { @@ -557,7 +557,7 @@ func (va ValidationAuthorityImpl) validateDNS(identifier core.AcmeIdentifier, in // Overall validation process -func (va ValidationAuthorityImpl) validate(authz core.Authorization, challengeIndex int, accountKey jose.JsonWebKey) { +func (va *ValidationAuthorityImpl) validate(authz core.Authorization, challengeIndex int, accountKey jose.JsonWebKey) { logEvent := verificationRequestEvent{ ID: authz.ID, Requester: authz.RegistrationID, @@ -603,7 +603,7 @@ func (va ValidationAuthorityImpl) validate(authz core.Authorization, challengeIn } // UpdateValidations runs the validate() method asynchronously using goroutines. -func (va ValidationAuthorityImpl) UpdateValidations(authz core.Authorization, challengeIndex int, accountKey jose.JsonWebKey) error { +func (va *ValidationAuthorityImpl) UpdateValidations(authz core.Authorization, challengeIndex int, accountKey jose.JsonWebKey) error { go va.validate(authz, challengeIndex, accountKey) return nil } From 469253a9e385ee2c966a8b7edede5789ae954e25 Mon Sep 17 00:00:00 2001 From: Jeff Hodges Date: Tue, 25 Aug 2015 21:42:48 -0700 Subject: [PATCH 4/5] fix some dregs in startservers.py Changes this to use just communicate(), not the subprocess.PIPE stuff (which apparently can do Weird Things) Also rename the install variable to cmd in the install function --- test/startservers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/startservers.py b/test/startservers.py index a10afc0be..c6bb962a3 100644 --- a/test/startservers.py +++ b/test/startservers.py @@ -33,13 +33,13 @@ processes = [] def install(progs, race_detection): - install = "go install" + cmd = "go install" if race_detection: - install = """go install -race""" - cmd = install + cmd = """go install -race""" + for prog in progs: cmd += " ./" + prog - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(cmd, shell=True) out, err = p.communicate() if p.returncode != 0: sys.stderr.write("unable to run go install: %s\n" % cmd) From 2f18259f47040ce4ba9381866d862cfb1ca22a7d Mon Sep 17 00:00:00 2001 From: Jeff Hodges Date: Tue, 25 Aug 2015 22:39:33 -0700 Subject: [PATCH 5/5] bump Dockerfile to Go 1.5 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2f09d4a6f..b6a22aa59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.5rc1 +FROM golang:1.5 MAINTAINER J.C. Jones "jjones@letsencrypt.org" MAINTAINER William Budington "bill@eff.org"