""" Script that automates trusted pull/pushes on different docker versions. """ from __future__ import print_function from collections import OrderedDict import atexit import json import os import platform import pwd import re import shutil import subprocess from tempfile import mkdtemp from time import sleep, time import urllib from urlparse import urljoin # Configuration for testing # please give the full path to the binary (or if it's on your path, just the # binary name) for these if you do not want them downloaded, otherwise these # can be ignored. Up to you to make sure you are running the correct daemon # version. DOCKERS = { "1.8": "docker-1.8.3", "1.9": "docker-1.9.1", "1.10": "docker", } # delete any of these if you want to specify the docker binaries yourself DOWNLOAD_DOCKERS = { "1.8": ("https://get.docker.com", "docker-1.8.3"), "1.9": ("https://get.docker.com", "docker-1.9.1"), "1.10": ("https://get.docker.com", "docker-1.10.3") } # please replace with private registry if you want to test against a private # registry REGISTRY = "docker.io" # please enter your username if it does not match your shell username, or set the # environment variable DOCKER_USERNAME REGISTRY_USERNAME = os.getenv("DOCKER_USERNAME", pwd.getpwuid(os.getuid())[0]) # what you want the testing repo names to be prefixed with REPO_PREFIX = "docker_test" # Assumes default docker config dir DEFAULT_DOCKER_CONFIG = os.path.expanduser("~/.docker") # Assumes the trust server will be run using compose if DOCKER_CONTENT_TRUST_SERVER is not specified DEFAULT_NOTARY_SERVER = "https://notary-server:4443" # please enter a custom trust server location if you do not wish to use a local # docker-compose instantiation. If testing against Docker Hub's notary server or # another trust server, please also ensure that this script does not pick up incorrect TLS # certificates from ~/.notary/config.json by default TRUST_SERVER = os.getenv('DOCKER_CONTENT_TRUST_SERVER', DEFAULT_NOTARY_SERVER) # Assumes the test will be run with `python misc/dockertest.py` from # the root of the notary repo after binaries are built # also overrides the notary server location if need be if TRUST_SERVER != DEFAULT_NOTARY_SERVER: NOTARY_CLIENT = "bin/notary -s {0}".format(TRUST_SERVER) else: NOTARY_CLIENT = "bin/notary -c cmd/notary/config.json" # ---- setup ---- def download_docker(download_dir="/tmp"): """ Downloads the relevant docker binaries and sets the docker values """ system = platform.system() architecture = "x86_64" if platform.architecture()[0] != "64bit": architecture = "i386" downloadfile = urllib.URLopener() for version in DOWNLOAD_DOCKERS: domain, binary = DOWNLOAD_DOCKERS[version] filename = os.path.join(download_dir, binary) if not os.path.isfile(filename): url = urljoin( domain, "/".join(["builds", system, architecture, binary])) print("Downloading", url) downloadfile.retrieve(url, filename) os.chmod(filename, 0755) DOCKERS[version] = filename def setup(): """ Ensure we are set up to run the test """ download_docker() # copy the docker config dir over so we don't break anything in real docker # config directory os.mkdir(_TEMP_DOCKER_CONFIG_DIR) # copy any docker creds over so we can push configfile = os.path.join(_TEMP_DOCKER_CONFIG_DIR, "config.json") shutil.copyfile(os.path.join(DEFAULT_DOCKER_CONFIG, "config.json"), configfile) # always clean up the config file so creds aren't left in this temp directory atexit.register(os.remove, configfile) defaulttlsdir = os.path.join(DEFAULT_DOCKER_CONFIG, "tls") tlsdir = os.path.join(_TEMP_DOCKER_CONFIG_DIR, "tls") if os.path.exists(tlsdir): shutil.copytree(defaulttlsdir, tlsdir) # make sure that the cert is in the right place for local notary if TRUST_SERVER == DEFAULT_NOTARY_SERVER: tlsdir = os.path.join(tlsdir, "notary-server:4443") if not os.path.isdir(tlsdir): try: shutil.rmtree(tlsdir) # in case it's not a directory except OSError as ex: if "No such file or directory" not in str(ex): raise os.makedirs(tlsdir) cert = os.path.join(tlsdir, "root-ca.crt") if not os.path.isfile(cert): shutil.copyfile("fixtures/root-ca.crt", cert) # ---- tests ---- _TEMPDIR = mkdtemp(prefix="docker-version-test") _TEMP_DOCKER_CONFIG_DIR = os.path.join(_TEMPDIR, "docker-config-dir") _TRUST_DIR = os.path.join(_TEMP_DOCKER_CONFIG_DIR, "trust") _ENV = os.environ.copy() _ENV.update({ # enable content trust and use our own server "DOCKER_CONTENT_TRUST_SERVER": TRUST_SERVER, "DOCKER_CONTENT_TRUST": "1", # environment variables that notary uses "NOTARY_ROOT_PASSPHRASE": "randompass", "NOTARY_TARGETS_PASSPHRASE": "randompass", "NOTARY_SNAPSHOT_PASSPHRASE": "randompass", # environment variables used by current version of docker "DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE": "randompass", "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE": "randompass", # environment variables used by docker 1.8 "DOCKER_CONTENT_TRUST_OFFLINE_PASSPHRASE": "randompass", "DOCKER_CONTENT_TRUST_TAGGING_PASSPHRASE": "randompass", # do not use the default docker config directory "DOCKER_CONFIG": _TEMP_DOCKER_CONFIG_DIR }) _DIGEST_REGEX = re.compile(r"\b[dD]igest: sha256:([0-9a-fA-F]+)\b") _SIZE_REGEX = re.compile(r"\bsize: ([0-9]+)\b") _PULL_A_REGEX = re.compile( r"Pull \(\d+ of \d+\): .+:(.+)@sha256:([0-9a-fA-F]+)") _BUILD_REGEX = re.compile(r"Successfully built ([0-9a-fA-F]+)") def clear_tuf(): """ Removes the trusted certificates and TUF metadata in ~/.docker/trust """ try: shutil.rmtree(os.path.join(_TRUST_DIR, "trusted_certificates")) shutil.rmtree(os.path.join(_TRUST_DIR, "tuf")) except OSError as ex: if "No such file or directory" not in str(ex): raise def clear_keys(): """ Removes the TUF keys in trust directory, since the key format changed between versions and can cause problems if testing newer docker versions before testing older docker versions. """ try: shutil.rmtree(os.path.join(_TRUST_DIR, "private")) except OSError as ex: if "No such file or directory" not in str(ex): raise def run_cmd(cmd, fileoutput): """ Takes a string command, runs it, and returns the output even if it fails. """ print("$ " + cmd) fileoutput.write("$ {0}\n".format(cmd)) try: output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, env=_ENV) except subprocess.CalledProcessError as ex: print(ex.output) fileoutput.write(ex.output) raise else: if output: print(output) fileoutput.write(output) return output finally: print() fileoutput.write("\n") def rmi(fout, docker_version, image, tag): """ Ensures that an image is no longer available locally to docker. """ try: run_cmd( "{0} rmi {1}:{2}".format(DOCKERS[docker_version], image, tag), fout) except subprocess.CalledProcessError as ex: if "could not find image" not in str(ex): raise def assert_equality(actual, expected): """ Assert equality, print nice message """ assert actual == expected, "\nGot : {0}\nExpected: {1}".format( repr(actual), repr(expected)) def pull(fout, docker_version, image, tag, expected_sha): """ Pulls an image using docker, and asserts that the sha is correct. Make sure it is untagged first. """ clear_tuf() rmi(fout, docker_version, image, tag) output = run_cmd("{0} pull {1}:{2}".format(DOCKERS[docker_version], image, tag), fout) sha = _DIGEST_REGEX.search(output).group(1) assert_equality(sha, expected_sha) def push(fout, docker_version, image, tag): """ Tags an image with the docker version and pushes it. Returns the sha and expected size. """ clear_tuf() # tag image with the docker version run_cmd( "{0} tag -f alpine {1}:{2}".format(DOCKERS[docker_version], image, tag), fout) # push! output = run_cmd("{0} push {1}:{2}".format(DOCKERS[docker_version], image, tag), fout) sha = _DIGEST_REGEX.search(output).group(1) size = _SIZE_REGEX.search(output).group(1) # sleep for 1s after pushing, just to let things propagate :) time.sleep(1) # list targets = notary_list(fout, image) for target in targets: if target[0] == tag: assert_equality(target, [tag, sha, size, "targets"]) return sha, size def notary_list(fout, repo): """ Calls notary list on the repo and returns a list of lists of tags, shas, sizes, and roles. """ clear_tuf() output = run_cmd( "{0} -d {1} list {2}".format(NOTARY_CLIENT, _TRUST_DIR, repo), fout) lines = output.strip().split("\n") assert len(lines) >= 3, "not enough targets" return [line.strip().split() for line in lines[2:]] def test_build(fout, image, docker_version): """ Build from a simple Dockerfile and ensure it works with DCT enabled """ clear_tuf() # build # simple dockerfile to test building with trust dockerfile = "FROM {0}:{1}\nRUN sh\n".format(image, docker_version) tempdir_dockerfile = os.path.join(_TEMPDIR, "Dockerfile") with open(tempdir_dockerfile, 'wb') as ftemp: ftemp.write(dockerfile) output = run_cmd( "{0} build {1}".format(DOCKERS[docker_version], _TEMPDIR), fout) build_result = _BUILD_REGEX.findall(output) assert len(build_result) >= 0, "build did not succeed" def test_pull_a(fout, docker_version, image, expected_tags): """ Pull -A on an image and ensure that all the expected tags are present """ clear_tuf() # remove every image possible for tag in expected_tags: rmi(fout, docker_version, image, tag) # pull -a output = run_cmd( "{0} pull -a {1}".format(DOCKERS[docker_version], image), fout) pulled_tags = _PULL_A_REGEX.findall(output) assert_equality(len(pulled_tags), len(expected_tags)) for tag, info in expected_tags.iteritems(): found = [pulled for pulled in pulled_tags if pulled[0] == tag] assert found assert_equality(found[0][1], info["sha"]) def test_push(tempdir, docker_version, image, tag="", allow_push_failure=False, do_after_first_push=None): """ Tests a push of an image by pushing with this docker version, and asserting that all the other docker versions can pull it. """ if not tag: tag = docker_version filename = os.path.join( tempdir, "{0}_{1}_push_{2}").format(time(), docker_version, tag) with open(filename, 'wb') as fout: try: sha, size = push(fout, docker_version, image, tag=tag) except subprocess.CalledProcessError: if allow_push_failure: return {"push": "failed, but that was expected"} raise return_val = { "push": { "sha": sha, "size": size } } if do_after_first_push is not None: do_after_first_push(fout, image) for ver in DOCKERS: try: pull(fout, ver, image, tag, sha) except subprocess.CalledProcessError: print("pulling {0}:{1} with {2} (expected hash {3}) failed".format( image, tag, ver, sha)) raise else: return_val["push"][ver] = "pull succeeded" return return_val def test_run(fout, image, docker_version): """ Runs a simple alpine container to ensure it works with DCT enabled """ clear_tuf() # run output = run_cmd( "{0} run -it --rm {1}:{2} echo SUCCESS".format(DOCKERS[docker_version], image, docker_version), fout) assert "SUCCESS" in output, "run did not succeed" def test_docker_version(docker_version, repo_name="", do_after_first_push=None): """ Initialize a repo with one docker version. Test that all other docker versions against that repo (both pulling and pushing). """ if not repo_name: repo_name = "repo_by_{0}".format(docker_version) tempdir = os.path.join(_TEMPDIR, repo_name) os.makedirs(tempdir) image = "{0}/{1}/{2}_{3}-{4}".format( REGISTRY, REGISTRY_USERNAME, REPO_PREFIX, repo_name, time()) result = OrderedDict([ (docker_version, test_push(tempdir, docker_version, image, do_after_first_push=do_after_first_push)) ]) # push again if we did something after the first push if do_after_first_push: tag = docker_version + "_push_again" result[tag] = test_push( tempdir, docker_version, image, tag=tag, # 1.8.x and 1.9.x might fail to push again after snapshot rotation # or delegation manipulation allow_push_failure=re.compile(r"1\.[0-9](\.\d+)?$").search(docker_version)) for ver in DOCKERS: if ver != docker_version: # 1.8.x and 1.9.x will fail to push if the repo was created by # a more recent docker, since the key format has changed, or if a # snapshot rotation or delegation has occurred can_fail = ( (do_after_first_push or re.compile(r"1\.[1-9][0-9](\.\d+)?$").search(docker_version)) and re.compile(r"1\.[0-9](\.\d+)?$").search(ver)) result[ver] = test_push(tempdir, ver, image, allow_push_failure=can_fail) # find all the successfully pushed tags expected_tags = {} for ver in result: if isinstance(result[ver]["push"], dict): expected_tags[ver] = result[ver]["push"] with open(os.path.join(tempdir, "pull_a"), 'wb') as fout: for ver in DOCKERS: try: test_pull_a(fout, ver, image, expected_tags) except subprocess.CalledProcessError: result[ver]["pull-a"] = "failed" else: result[ver]["pull-a"] = "success" with open(os.path.join(tempdir, "notary_list"), 'wb') as fout: targets = notary_list(fout, image) assert_equality(len(targets), len(expected_tags)) for tag, info in expected_tags.iteritems(): found = [target for target in targets if target[0] == tag] assert found assert_equality( found[0][1:], [info["sha"], info["size"], "targets"]) result["list"] = "listed expected targets successfully" with open(os.path.join(tempdir, "build"), 'wb') as fout: try: test_build(fout, image, docker_version) except subprocess.CalledProcessError: result[docker_version]["build"] = "failed" else: result[docker_version]["build"] = "success" with open(os.path.join(tempdir, "run"), 'wb') as fout: try: test_run(fout, image, docker_version) except subprocess.CalledProcessError: result[docker_version]["run"] = "failed" else: result[docker_version]["run"] = "success" with open(os.path.join(tempdir, "result.json"), 'wb') as fout: json.dump(result, fout, indent=2) return result def rotate_to_server_snapshot(fout, image): """ Uses the notary client to rotate the snapshot key to be server-managed. """ run_cmd( "{0} -d {1} key rotate {2} snapshot -r".format( NOTARY_CLIENT, _TRUST_DIR, image), fout) run_cmd( "{0} -d {1} publish {2}".format(NOTARY_CLIENT, _TRUST_DIR, image), fout) def test_all_docker_versions(): """ Initialize a repo with each docker version, and test that other docker versions can read/write to it. """ print("Output files at", _TEMPDIR) results = OrderedDict() for docker_version in DOCKERS: clear_keys() # test with just creating a regular repo result = test_docker_version(docker_version) print("\nRepo created with docker {0}:".format(docker_version)) print(json.dumps(result, indent=2)) results[docker_version] = result # do snapshot rotation after creating the repo, and see if it's still ok repo_name = "repo_by_{0}_snapshot_rotation".format(docker_version) result = test_docker_version( docker_version, repo_name=repo_name, do_after_first_push=rotate_to_server_snapshot) print("\nRepo created with docker {0} and snapshot key rotated:" .format(docker_version)) print(json.dumps(result, indent=2)) results[docker_version + "_snapshot_rotation"] = result with open(os.path.join(_TEMPDIR, "total_results.json"), 'wb') as fout: json.dump(results, fout, indent=2) print("\n\nFinal results:") results["output_dir"] = _TEMPDIR print(json.dumps(results, indent=2)) if __name__ == "__main__": setup() test_all_docker_versions()