Improve integration tests. (#4847)

Add passthrough for certain environment variables to
docker-compose.yml, making it easier to set them:

    RUN=unit docker-compose run --use-aliases boulder ./test.sh

Use 4001 instead of 4443 to monitor boulder-wfe2's health. This avoids
a spurious error log about a failed TLS handshake.

Remove unused code around running Certbot in integation tests.

docker-compose.yml

@@ -4,13 +4,18 @@ services:
         # To minimize fetching this should be the same version used below
         image: letsencrypt/boulder-tools-go${TRAVIS_GO_VERSION:-1.14.1}:2020-06-01
         environment:
-            FAKE_DNS: 10.77.77.77
-            BOULDER_CONFIG_DIR: test/config
-            GO111MODULE: "on"
-            GOFLAGS: "-mod=vendor"
+            - FAKE_DNS=10.77.77.77
+            - BOULDER_CONFIG_DIR=test/config
+            - GOFLAGS=-mod=vendor
             # This is required so Python doesn't throw an error when printing
             # non-ASCII to stdout.
-            PYTHONIOENCODING: "utf-8"
+            - PYTHONIOENCODING=utf-8
+            # These are variables you can set to affect what tests get run or
+            # how they are run. Including them here with no value means they are
+            # passed through from the environment.
+            - RUN
+            - INT_FILTER
+            - RACE
         volumes:
           - .:/go/src/github.com/letsencrypt/boulder:cached
           - ./.gocache:/root/.cache/go-build:cached

test/integration-test.py

@@ -38,14 +38,6 @@ race_detection = True
 if os.environ.get('RACE', 'true') != 'true':
     race_detection = False
 
-def run_client_tests():
-    root = os.environ.get("CERTBOT_PATH")
-    assert root is not None, (
-        "Please set CERTBOT_PATH env variable to point at "
-        "initialized (virtualenv) client repo root")
-    cmd = os.path.join(root, 'tests', 'boulder-integration.sh')
-    run(cmd, cwd=root)
-
 def run_go_tests(filterPattern=None):
     """
     run_go_tests launches the Go integration tests. The go test command must
@@ -243,8 +235,6 @@ exit_status = 1
 
 def main():
     parser = argparse.ArgumentParser(description='Run integration tests')
-    parser.add_argument('--certbot', dest='run_certbot', action='store_true',
-                        help="run the certbot integration tests")
     parser.add_argument('--chisel', dest="run_chisel", action="store_true",
                         help="run integration tests using chisel")
     parser.add_argument('--gotest', dest="run_go", action="store_true",
@@ -254,12 +244,11 @@ def main():
     # allow any ACME client to run custom command for integration
     # testing (without having to implement its own busy-wait loop)
     parser.add_argument('--custom', metavar="CMD", help="run custom command")
-    parser.set_defaults(run_certbot=False, run_chisel=False,
-        test_case_filter="", skip_setup=False)
+    parser.set_defaults(run_chisel=False, test_case_filter="", skip_setup=False)
     args = parser.parse_args()
 
-    if not (args.run_certbot or args.run_chisel or args.custom  or args.run_go is not None):
-        raise(Exception("must run at least one of the letsencrypt or chisel tests with --certbot, --chisel, --gotest, or --custom"))
+    if not (args.run_chisel or args.custom  or args.run_go is not None):
+        raise(Exception("must run at least one of the letsencrypt or chisel tests with --chisel, --gotest, or --custom"))
 
     # Setup issuance hierarchy
     startservers.setupHierarchy()
@@ -286,9 +275,6 @@ def main():
     if args.run_chisel:
         run_chisel(args.test_case_filter)
 
-    if args.run_certbot:
-        run_client_tests()
-
     if args.run_go:
         run_go_tests(args.test_case_filter)
 

test/startservers.py

@@ -91,7 +91,7 @@ def start(race_detection, fakeclock):
         [8102, './bin/boulder-ra --config %s --addr ra2.boulder:9094 --debug-addr :8102' % os.path.join(config_dir, "ra.json")],
         [8111, './bin/nonce-service --config %s --addr nonce1.boulder:9101 --debug-addr :8111 --prefix taro' % os.path.join(config_dir, "nonce.json")],
         [8112, './bin/nonce-service --config %s --addr nonce2.boulder:9101 --debug-addr :8112 --prefix zinc' % os.path.join(config_dir, "nonce.json")],
-        [4431, './bin/boulder-wfe2 --config %s' % os.path.join(config_dir, "wfe2.json")],
+        [4001, './bin/boulder-wfe2 --config %s' % os.path.join(config_dir, "wfe2.json")],
         [4000, './bin/boulder-wfe --config %s' % os.path.join(config_dir, "wfe.json")],
         [8016, './bin/log-validator --config %s' % os.path.join(config_dir, "log-validator.json")],
     ])