diff --git a/test.sh b/test.sh index 25c6bdaee..b9dc66e97 100755 --- a/test.sh +++ b/test.sh @@ -191,7 +191,7 @@ if [[ "$RUN" =~ "integration" ]] ; then source ${CERTBOT_PATH}/${VENV_NAME:-venv}/bin/activate fi - run python test/integration-test.py --all + run python test/integration-test.py --chisel end_context #integration fi diff --git a/test/chisel.py b/test/chisel.py new file mode 100644 index 000000000..b5f82eb8e --- /dev/null +++ b/test/chisel.py @@ -0,0 +1,178 @@ +""" +A simple client that uses the Python ACME library to run a test issuance against +a local Boulder server. Usage: + +$ virtualenv venv +$ . venv/bin/activate +$ pip install -r requirements.txt +$ python chisel.py foo.com bar.com +""" +import json +import logging +import os +import sys +import threading +import time +import urllib2 + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa + +import OpenSSL + +from acme import challenges +from acme import client as acme_client +from acme import errors as acme_errors +from acme import jose +from acme import messages +from acme import standalone + +logger = logging.getLogger() +logger.setLevel(int(os.getenv('LOGLEVEL', 20))) + +def make_client(email=None): + """Build an acme.Client and register a new account with a random key.""" + key = jose.JWKRSA(key=rsa.generate_private_key(65537, 2048, default_backend())) + + net = acme_client.ClientNetwork(key, verify_ssl=False, + user_agent="Boulder integration tester") + + client = acme_client.Client("http://localhost:4000/directory", key=key, net=net) + account = client.register(messages.NewRegistration.from_data(email=email)) + client.agree_to_tos(account) + return client + +def get_chall(client, domain): + """Ask the server for an authz, return the authz and an HTTP-01 challenge.""" + authz = client.request_domain_challenges(domain) + for chall_body in authz.body.challenges: + if isinstance(chall_body.chall, challenges.HTTP01): + return authz, chall_body + raise "No HTTP-01 challenge found" + +def make_authzs(client, domains): + """Make authzs for each of the given domains. Return a list of authzs + and challenges.""" + authzs, challenges = [], [] + for d in domains: + authz, chall_body = get_chall(client, d) + + authzs.append(authz) + challenges.append(chall_body) + return authzs, challenges + +class ValidationError(Exception): + """An error that occurs during challenge validation.""" + def __init__(self, domain, problem_type, detail, *args, **kwargs): + self.domain = domain + self.problem_type = problem_type + self.detail = detail + + def __str__(self): + return "%s: %s: %s" % (self.domain, self.problem_type, self.detail) + +def issue(client, authzs, cert_output=None): + """Given a list of authzs that are being processed by the server, + wait for them to be ready, then request issuance of a cert with a random + key for the given domains.""" + domains = [authz.body.identifier.value for authz in authzs] + pkey = OpenSSL.crypto.PKey() + pkey.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) + csr = OpenSSL.crypto.X509Req() + csr.add_extensions([ + OpenSSL.crypto.X509Extension( + 'subjectAltName', + critical=False, + value=', '.join('DNS:' + d for d in domains).encode() + ), + ]) + csr.set_pubkey(pkey) + csr.set_version(2) + csr.sign(pkey, 'sha256') + + cert_resource = None + try: + cert_resource, _ = client.poll_and_request_issuance(jose.ComparableX509(csr), authzs) + except acme_errors.PollError as error: + # If we get a PollError, pick the first failed authz and turn it into a more + # useful ValidationError that contains details we can look for in tests. + for authz in error.updated: + updated_authz = json.loads(urllib2.urlopen(authz.uri).read()) + domain = authz.body.identifier.value, + for c in updated_authz['challenges']: + if 'error' in c: + err = c['error'] + raise ValidationError(domain, err['type'], err['detail']) + # If none of the authz's had an error, just re-raise. + raise + if cert_output != None: + pem = OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, + cert_resource.body) + with open(cert_output, 'w') as f: + f.write(pem) + return cert_resource + +def http_01_answer(client, chall_body): + """Return an HTTP01Resource to server in response to the given challenge.""" + response, validation = chall_body.response_and_validation(client.key) + return standalone.HTTP01RequestHandler.HTTP01Resource( + chall=chall_body.chall, response=response, + validation=validation) + +def auth_and_issue(domains, email=None, cert_output=None, client=None): + """Make authzs for each of the given domains, set up a server to answer the + challenges in those authzs, tell the ACME server to validate the challenges, + then poll for the authzs to be ready and issue a cert.""" + if client == None: + client = make_client(email) + authzs, challenges = make_authzs(client, domains) + port = 5002 + answers = set([http_01_answer(client, c) for c in challenges]) + server = standalone.HTTP01Server(("", port), answers) + thread = threading.Thread(target=server.serve_forever) + thread.start() + + # Loop until the HTTP01Server is ready. + while True: + try: + urllib2.urlopen("http://localhost:%d" % port) + break + except urllib2.URLError: + time.sleep(0.1) + + try: + for chall_body in challenges: + client.answer_challenge(chall_body, chall_body.response(client.key)) + cert_resource = issue(client, authzs, cert_output) + return cert_resource + finally: + server.shutdown() + server.server_close() + thread.join() + +def expect_problem(problem_type, func): + """Run a function. If it raises a ValidationError or messages.Error that + contains the given problem_type, return. If it raises no error or the wrong + error, raise an exception.""" + ok = False + try: + func() + except ValidationError as e: + if e.problem_type == problem_type: + ok = True + else: + raise + except messages.Error as e: + if problem_type in e.__str__(): + ok = True + else: + raise + if not ok: + raise Exception("Expected %s, got no error" % problem_type) + +if __name__ == "__main__": + try: + auth_and_issue(sys.argv[1:]) + except messages.Error, e: + print e + sys.exit(1) diff --git a/test/integration-test.py b/test/integration-test.py index 5be1bead8..8d75a44b9 100644 --- a/test/integration-test.py +++ b/test/integration-test.py @@ -3,30 +3,27 @@ import argparse import atexit import base64 import datetime +import errno import json import os import random import re import shutil -import socket import subprocess +import signal import sys import tempfile import time -import urllib import urllib2 import startservers -ISSUANCE_FAILED = 1 -REVOCATION_FAILED = 2 -MAILER_FAILED = 3 +import chisel +from chisel import auth_and_issue class ExitStatus: OK, PythonFailure, NodeFailure, Error, OCSPFailure, CTFailure, IncorrectCommandLineArgs, RevokerFailure = range(8) -JS_DIR = 'test/js' - class ProcInfo: """ Args: @@ -55,7 +52,7 @@ def fetch_ocsp(request_bytes, url): # Make the OCSP request three different ways: by POST, by GET, and by GET with # URL-encoded parameters. All three should have an identical response. get_response = urllib2.urlopen("%s/%s" % (url, ocsp_req_b64)).read() - get_encoded_response = urllib2.urlopen("%s/%s" % (url, urllib.quote(ocsp_req_b64, safe = ""))).read() + get_encoded_response = urllib2.urlopen("%s/%s" % (url, urllib2.quote(ocsp_req_b64, safe = ""))).read() post_response = urllib2.urlopen("%s/" % (url), request_bytes).read() return (post_response, get_response, get_encoded_response) @@ -151,39 +148,12 @@ def wait_for_ocsp_good(cert_file, issuer_file, url): def wait_for_ocsp_revoked(cert_file, issuer_file, url): fetch_until(cert_file, issuer_file, url, ": good", ": revoked") -def get_expiry_time(cert_file): - try: - output = subprocess.check_output(["openssl", "x509", "-enddate", "-noout", "-in", cert_file]) - except subprocess.CalledProcessError as e: - output = e.output - print output - print "subprocess returned non-zero: %s" % e - die(ExitStatus.NodeFailure) +def test_multidomain(): + auth_and_issue([random_domain(), random_domain()]) - return datetime.datetime.strptime(output.split('\n')[0].split('=')[1], '%b %d %H:%M:%S %Y %Z') - -def verify_ct_submission(expectedSubmissions, url): - resp = urllib2.urlopen(url) - submissionStr = resp.read() - if int(submissionStr) != expectedSubmissions: - print "Expected %d submissions, found %d" % (expectedSubmissions, int(submissionStr)) - die(ExitStatus.CTFailure) - return 0 - -def run_node_test(domain, chall_type, expected_ct_submissions): - email_addr = "js.integration.test@letsencrypt.org" - cert_file = os.path.join(tempdir, "cert.der") +def test_ocsp(): cert_file_pem = os.path.join(tempdir, "cert.pem") - key_file = os.path.join(tempdir, "key.pem") - # Issue the certificate and transform it from DER-encoded to PEM-encoded. - if subprocess.Popen(''' - node test.js --email %s --domains %s \ - --certKey %s --cert %s --challType %s && \ - openssl x509 -in %s -out %s -inform der -outform pem - ''' % (email_addr, domain, key_file, cert_file, chall_type, cert_file, cert_file_pem), - shell=True, cwd=JS_DIR).wait() != 0: - print("\nIssuing failed") - return ISSUANCE_FAILED + auth_and_issue([random_domain()], cert_output=cert_file_pem) ee_ocsp_url = "http://localhost:4002" @@ -191,40 +161,65 @@ def run_node_test(domain, chall_type, expected_ct_submissions): # checking OCSP until we either see a good response or we timeout (5s). wait_for_ocsp_good(cert_file_pem, "test/test-ca2.pem", ee_ocsp_url) - verify_ct_submission(expected_ct_submissions, "http://localhost:4500/submissions") +def test_ct_submission(): + url = "http://localhost:4500/submissions" + submissions = urllib2.urlopen(url).read() + expected_submissions = int(submissions)+1 + auth_and_issue([random_domain()]) + submissions = urllib2.urlopen(url).read() + if int(submissions) != expected_submissions: + print "Expected %d submissions, found %s" % (expected_submissions, submissions) + die(ExitStatus.CTFailure) +def random_domain(): + """Generate a random domain for testing (to avoid rate limiting).""" + return "rand.%x.xyz" % random.randrange(2**32) + +def test_expiration_mailer(): + email_addr = "integration.%x@boulder.local" % random.randrange(2**16) + cert = auth_and_issue([random_domain()], email=email_addr).body # Check that the expiration mailer sends a reminder - expiry = get_expiry_time(cert_file_pem) + expiry = datetime.datetime.strptime(cert.get_notAfter(), '%Y%m%d%H%M%SZ') no_reminder = expiry + datetime.timedelta(days=-31) first_reminder = expiry + datetime.timedelta(days=-13) last_reminder = expiry + datetime.timedelta(days=-2) try: urllib2.urlopen("http://localhost:9381/clear", data='') - get_future_output('./bin/expiration-mailer --config %s/expiration-mailer.json' % + print get_future_output('./bin/expiration-mailer --config %s/expiration-mailer.json' % default_config_dir, no_reminder) - get_future_output('./bin/expiration-mailer --config %s/expiration-mailer.json' % + print get_future_output('./bin/expiration-mailer --config %s/expiration-mailer.json' % default_config_dir, first_reminder) - get_future_output('./bin/expiration-mailer --config %s/expiration-mailer.json' % + print get_future_output('./bin/expiration-mailer --config %s/expiration-mailer.json' % default_config_dir, last_reminder) resp = urllib2.urlopen("http://localhost:9381/count?to=%s" % email_addr) mailcount = int(resp.read()) if mailcount != 2: print("\nExpiry mailer failed: expected 2 emails, got %d" % mailcount) - return MAILER_FAILED + die(1) except Exception as e: print("\nExpiry mailer failed:") print(e) - return MAILER_FAILED + die(1) - if subprocess.Popen(''' - node revoke.js %s %s http://localhost:4000/acme/revoke-cert - ''' % (cert_file, key_file), shell=True, cwd=JS_DIR).wait() != 0: - print("\nRevoking failed") - return REVOCATION_FAILED +def test_revoke_by_account(): + cert_file_pem = os.path.join(tempdir, "revokeme.pem") + client = chisel.make_client() + cert = auth_and_issue([random_domain()], client=client).body + client.revoke(cert.body) wait_for_ocsp_revoked(cert_file_pem, "test/test-ca2.pem", ee_ocsp_url) return 0 +def test_caa(): + """Request issuance for two CAA domains, one where we are permitted and one where we are not.""" + auth_and_issue(["good-caa-reserved.com"]) + + # TODO(#2514): Currently, the gRPC setup doesn't correctly set the error + # field on failed validations. Once #2514 is fixed, remove this if statement. + if os.getenv('BOULDER_CONFIG_DIR') != 'test/config-next': + chisel.expect_problem("urn:acme:error:connection", + lambda: auth_and_issue(["bad-caa-reserved.com"])) + def run_custom(cmd, cwd=None): if subprocess.Popen(cmd, shell=True, cwd=cwd, executable='/bin/bash').wait() != 0: die(ExitStatus.PythonFailure) @@ -256,20 +251,18 @@ def single_ocsp_sign(): p = subprocess.Popen( './bin/ocsp-responder --config test/issuer-ocsp-responder.json', shell=True) - global ocsp_proc - ocsp_proc = p - # Verify that the static OCSP responder, which answers with a # pre-signed, long-lived response for the CA cert, works. wait_for_ocsp_good("test/test-ca2.pem", "test/test-root.pem", "http://localhost:4003") + p.send_signal(signal.SIGTERM) + def get_future_output(cmd, date, cwd=None): return subprocess.check_output(cmd, cwd=cwd, env={'FAKECLOCK': date.strftime("%a %b %d %H:%M:%S UTC %Y")}, shell=True) -def run_expired_authz_purger_test(): - subprocess.check_output('''node test.js --email %s --domains %s --abort-step %s''' % - ("purger@test.com", "eap-test.com", "startChallenge"), - shell=True, cwd=JS_DIR) +def test_expired_authz_purger(): + # Make an authz, but don't attempt its challenges. + chisel.make_client().request_domain_challenges("eap-test.com") def expect(target_time, num): expected_output = 'Deleted a total of %d expired pending authorizations' % num @@ -288,80 +281,36 @@ def run_expired_authz_purger_test(): expect(now, 0) expect(after_grace_period, 1) -def run_certificates_per_name_test(): - try: - # This command will return a non zero error code. In order - # to avoid a CalledProcessException we use Popen. - handle = subprocess.Popen( - '''node test.js --email %s --domains %s''' % ('test@lim.it', 'lim.it'), - shell=True, cwd=JS_DIR, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - handle.wait() - out, err = handle.communicate() - except subprocess.CalledProcessError as e: - print("\nFailure while running certificates per name test %s" % e) - die(ExitStatus.PythonFailure) - - expected = [ - "urn:acme:error:rateLimited", - "Error creating new cert :: Too many certificates already issued for: lim.it", - "429" - ] - for s in expected: - if s not in out: - print("\nCertificates per name test: expected %s not present in output" % s) - die(ExitStatus.Error) +def test_certificates_per_name(): + chisel.expect_problem("urn:acme:error:rateLimited", + lambda: auth_and_issue(["lim.it"])) default_config_dir = os.environ.get('BOULDER_CONFIG_DIR', '') if default_config_dir == '': default_config_dir = 'test/config' -def run_admin_revoker_test(): - cert_file = os.path.join(tempdir, "ar-cert.der") +def test_admin_revoker_cert(): cert_file_pem = os.path.join(tempdir, "ar-cert.pem") - # Issue certificate for serial-revoke test - if subprocess.Popen(''' - node test.js --domains ar-test.com --cert %s && \ - openssl x509 -in %s -out %s -inform der -outform pem - ''' % (cert_file, cert_file, cert_file_pem), - shell=True, cwd=JS_DIR).wait() != 0: - print("\nIssuing failed") - die(ExitStatus.NodeFailure) - # Extract serial from certificate - try: - serial = subprocess.check_output("openssl x509 -in %s -noout -serial | cut -c 8-" % (cert_file_pem), - shell=True, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - print "Failed to extract serial: %s" % (e.output) - die(ExitStatus.PythonFailure) - serial = serial.rstrip() + cert = auth_and_issue([random_domain()], cert_output=cert_file_pem).body + serial = "%x" % cert.get_serial_number() # Revoke certificate by serial - config = default_config_dir + "/admin-revoker.json" - if subprocess.Popen("./bin/admin-revoker serial-revoke --config %s %s %d" % (config, serial, 1), - shell=True).wait() != 0: + if subprocess.Popen( + "./bin/admin-revoker serial-revoke --config %s/admin-revoker.json %s %d" % ( + default_config_dir, serial, 1), shell=True).wait() != 0: print("Failed to revoke certificate") die(ExitStatus.RevokerFailure) # Wait for OCSP response to indicate revocation took place ee_ocsp_url = "http://localhost:4002" wait_for_ocsp_revoked(cert_file_pem, "test/test-ca2.pem", ee_ocsp_url) - # Issue certificate for auth-revoke test - try: - output = subprocess.check_output("node test.js --domains ar-auth-test.com --abort-step startChallenge", - shell=True, cwd=JS_DIR) - except subprocess.CalledProcessError as e: - print "Failed to create authorization: %s" % (e.output) - die(ExitStatus.NodeFailure) - # Get authorization URL from last line of output - lines = output.rstrip().split("\n") - prefix = "authorization-url=" - if not lines[-1].startswith(prefix): - print("Failed to extract authorization URL") - die(ExitStatus.NodeFailure) - url = lines[-1][len(prefix):] +def test_admin_revoker_authz(): + # Make an authz, but don't attempt its challenges. + authz_resource = chisel.make_client().request_domain_challenges("ar-auth-test.com") + url = authz_resource.uri # Revoke authorization by domain try: - output = subprocess.check_output("./bin/admin-revoker auth-revoke --config %s ar-auth-test.com" % (config), - shell=True) + output = subprocess.check_output( + "./bin/admin-revoker auth-revoke --config %s/admin-revoker.json ar-auth-test.com" % (default_config_dir), shell=True) except subprocess.CalledProcessError as e: print("Failed to revoke authorization: %s", e) die(ExitStatus.RevokerFailure) @@ -369,7 +318,7 @@ def run_admin_revoker_test(): print("admin-revoker didn't revoke the expected number of pending and finalized authorizations") die(ExitStatus.RevokerFailure) # Check authorization has actually been revoked - response = urllib.urlopen(url) + response = urllib2.urlopen(url) data = json.loads(response.read()) if data['status'] != "revoked": print("Authorization wasn't revoked") @@ -384,58 +333,35 @@ def main(): help="run all of the clients' integration tests") parser.add_argument('--certbot', dest='run_certbot', action='store_true', help="run the certbot integration tests") - parser.add_argument('--node', dest="run_node", action="store_true", - help="run the node client's integration tests") + parser.add_argument('--chisel', dest="run_chisel", action="store_true", + help="run integration tests using chisel") # allow any ACME client to run custom command for integration # testing (without having to implement its own busy-wait loop) parser.add_argument('--custom', metavar="CMD", help="run custom command") parser.set_defaults(run_all=False, run_certbot=False, run_node=False) args = parser.parse_args() - if not (args.run_all or args.run_certbot or args.run_node or args.custom is not None): - print >> sys.stderr, "must run at least one of the letsencrypt or node tests with --all, --certbot, --node, or --custom" + if not (args.run_all or args.run_certbot or args.run_chisel or args.custom is not None): + print >> sys.stderr, "must run at least one of the letsencrypt or node tests with --all, --certbot, --chisel, or --custom" die(ExitStatus.IncorrectCommandLineArgs) - if not startservers.start(race_detection=True): - die(ExitStatus.Error) - - single_ocsp_sign() - - if args.run_all or args.run_node: - if subprocess.Popen('npm install', shell=True, cwd=JS_DIR).wait() != 0: - print("\n Installing NPM modules failed") + # Keep track of whether we started the Boulder servers and need to shut them down. + started_servers = False + # Check if WFE is already running. + try: + urllib2.urlopen("http://localhost:4000/directory") + except urllib2.URLError: + # WFE not running, start all of Boulder. + started_servers = True + if not startservers.start(race_detection=True): die(ExitStatus.Error) - # Pick a random hostname so we don't run into certificate rate limiting. - domains = "www.%x-TEST.com,%x-test.com" % ( - random.randrange(2**32), random.randrange(2**32)) - challenge_types = ["http-01", "dns-01"] - expected_ct_submissions = 1 - resp = urllib2.urlopen("http://localhost:4500/submissions") - submissionStr = resp.read() - if int(submissionStr) > 0: - expected_ct_submissions = int(submissionStr)+1 - for chall_type in challenge_types: - if run_node_test(domains, chall_type, expected_ct_submissions) != 0: - die(ExitStatus.NodeFailure) - expected_ct_submissions += 1 - - if run_node_test("good-caa-reserved.com", challenge_types[0], expected_ct_submissions) != 0: - print("\nDidn't issue certificate for domain with good CAA records") - die(ExitStatus.NodeFailure) - - if run_node_test("bad-caa-reserved.com", challenge_types[0], expected_ct_submissions) != ISSUANCE_FAILED: - print("\nIssued certificate for domain with bad CAA records") - die(ExitStatus.NodeFailure) - - run_expired_authz_purger_test() - - run_certificates_per_name_test() - - run_admin_revoker_test() + if args.run_all or args.run_chisel: + run_chisel() # Simulate a disconnection from RabbitMQ to make sure reconnects work. - startservers.bounce_forward() + if started_servers: + startservers.bounce_forward() if args.run_all or args.run_certbot: run_client_tests() @@ -443,10 +369,24 @@ def main(): if args.custom: run_custom(args.custom) - if not startservers.check(): + if started_servers and not startservers.check(): die(ExitStatus.Error) exit_status = ExitStatus.OK +def run_chisel(): + # XXX: Test multiple challenge types + + test_expired_authz_purger() + test_multidomain() + test_expiration_mailer() + test_ct_submission() + test_caa() + test_admin_revoker_cert() + test_admin_revoker_authz() + test_certificates_per_name() + test_ocsp() + single_ocsp_sign() + if __name__ == "__main__": try: main() @@ -463,5 +403,3 @@ def stop(): else: if exit_status: print("\n\nFAILURE %d" % exit_status) - if ocsp_proc.poll() is None: - ocsp_proc.kill() diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 000000000..2bf757293 --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,3 @@ +acme>=0.10.1 +cryptography>=0.7 +PyOpenSSL