From 6a455081073e709b6c45d9a17775c053d248f6f2 Mon Sep 17 00:00:00 2001 From: Ying Li Date: Tue, 5 Jan 2016 17:55:32 -0800 Subject: [PATCH] Add a miscellaneous script for testing docker trust functions with notary. Signed-off-by: Ying Li --- misc/dockertest.py | 408 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 misc/dockertest.py diff --git a/misc/dockertest.py b/misc/dockertest.py new file mode 100644 index 0000000000..573302da31 --- /dev/null +++ b/misc/dockertest.py @@ -0,0 +1,408 @@ +""" +Script that automates trusted pull/pushes on different docker versions. +""" + +from __future__ import print_function +from collections import OrderedDict +import json +import os +import platform +import pwd +import re +import shutil +import subprocess +from tempfile import mkdtemp +from time import time +import urllib +from urlparse import urljoin + +# Configuration for testing + +# please give the full path to the binary (or if it's on your path, just the +# binary name) for these if you do not want them downloaded, otherwise these +# can be ignored. Up to you to make sure you are running the correct daemon +# version. +DOCKERS = { + "1.8": "docker-1.8.3", + "1.9": "docker", + "1.10": "docker-1.10.0-dev" +} + +# delete any of these if you want to specify the docker binaries yourself +DOWNLOAD_DOCKERS = { + "1.8": ("https://get.docker.com", "docker-1.8.3"), + "1.9": ("https://get.docker.com", "docker-1.9.1"), + # "1.10": ("https://experimental.docker.com", "docker-1.10.0-dev") +} + +# please replace with private registry if you want to test against a private +# registry +REGISTRY = "docker.io" + +# please enter your username if it does not match your shell username +REGISTRY_USERNAME = pwd.getpwuid(os.getuid())[0] + +# what you want the testing repo names to be prefixed with +REPO_PREFIX = "docker_test" + +# Assumes default docker trust dir +TRUST_DIR = os.path.expanduser("~/.docker/trust") + +# Assumes the test will be run with `python misc/dockertest.py` from +# the root of the notary repo after binaries are built +NOTARY_CLIENT = "bin/notary -c cmd/notary/config.json" + +# Assumes the trust server will be run using compose +TRUST_SERVER = "https://notary-server:4443" + +# ---- setup ---- + +def download_docker(download_dir="/tmp"): + """ + Downloads the relevant docker binaries and sets the docker values + """ + system = platform.system() + architecture = "x86_64" + if platform.architecture()[0] != "64bit": + architecture = "i386" + + downloadfile = urllib.URLopener() + for version in DOWNLOAD_DOCKERS: + domain, binary = DOWNLOAD_DOCKERS[version] + filename = os.path.join(download_dir, binary) + if not os.path.isfile(filename): + url = urljoin( + domain, "/".join(["builds", system, architecture, binary])) + print("Downloading", url) + downloadfile.retrieve(url, filename) + + os.chmod(filename, 0755) + DOCKERS[version] = filename + + +def setup(): + """ + Ensure we are set up to run the test + """ + download_docker() + # make sure that the cert is in the right place + if TRUST_SERVER == "https://notary-server:4443": + tlsdir = os.path.join(TRUST_DIR, "tls", "notary-server:4443") + if not os.path.isdir(tlsdir): + shutil.rmtree(tlsdir) # in case it's not a directory + os.makedirs(tlsdir) + cert = os.path.join(tlsdir, "root-ca.crt") + if not os.path.isfile(cert): + shutil.copyfile("fixtures/root-ca.crt", cert) + +# ---- tests ---- + +_ENV = os.environ.copy() +_ENV.update({ + # enable content trust and use our own server + "DOCKER_CONTENT_TRUST_SERVER": TRUST_SERVER, + "DOCKER_CONTENT_TRUST": "1", + + # environment variables that notary uses + "NOTARY_ROOT_PASSPHRASE": "randompass", + "NOTARY_TARGETS_PASSPHRASE": "randompass", + "NOTARY_SNAPSHOT_PASSPHRASE": "randompass", + + # environment variables used by current version of docker + "DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE": "randompass", + "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE": "randompass", + + # environment variables used by docker 1.8 + "DOCKER_CONTENT_TRUST_OFFLINE_PASSPHRASE": "randompass", + "DOCKER_CONTENT_TRUST_TAGGING_PASSPHRASE": "randompass" +}) + +_TEMPDIR = mkdtemp(prefix="docker-version-test") + +_DIGEST_REGEX = re.compile(r"\b[dD]igest: sha256:([0-9a-fA-F]+)\b") +_SIZE_REGEX = re.compile(r"\bsize: ([0-9]+)\b") +_PULL_A_REGEX = re.compile( + r"Pull \(\d+ of \d+\): .+:(.+)@sha256:([0-9a-fA-F]+)") + + +def clear_tuf(): + """ + Removes the trusted certificates and TUF metadata in ~/.docker/trust + """ + try: + shutil.rmtree(os.path.join(TRUST_DIR, "trusted_certificates")) + shutil.rmtree(os.path.join(TRUST_DIR, "tuf")) + except OSError as ex: + if "No such file or directory" not in str(ex): + raise + + +def run_cmd(cmd, fileoutput): + """ + Takes a string command, runs it, and returns the output even if it fails. + """ + print("$ " + cmd) + fileoutput.write("$ {0}\n".format(cmd)) + try: + output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, + env=_ENV) + except subprocess.CalledProcessError as ex: + print(ex.output) + fileoutput.write(ex.output) + raise + else: + if output: + print(output) + fileoutput.write(output) + return output + finally: + print() + fileoutput.write("\n") + + +def rmi(fout, docker_version, image, tag): + """ + Ensures that an image is no longer available locally to docker. + """ + try: + run_cmd( + "{0} rmi {1}:{2}".format(DOCKERS[docker_version], image, tag), + fout) + except subprocess.CalledProcessError as ex: + if "could not find image" not in str(ex): + raise + +def assert_equality(actual, expected): + """ + Assert equality, print nice message + """ + assert actual == expected, "\nGot : {0}\nExpected: {1}".format( + repr(actual), repr(expected)) + + +def pull(fout, docker_version, image, tag, expected_sha): + """ + Pulls an image using docker, and asserts that the sha is correct. Make + sure it is untagged first. + """ + clear_tuf() + rmi(fout, docker_version, image, tag) + output = run_cmd("{0} pull {1}:{2}".format(DOCKERS[docker_version], + image, tag), + fout) + sha = _DIGEST_REGEX.search(output).group(1) + assert_equality(sha, expected_sha) + + +def push(fout, docker_version, image, tag): + """ + Tags an image with the docker version and pushes it. Returns the sha and + expected size. + """ + clear_tuf() + + # tag image with the docker version + run_cmd( + "{0} tag -f alpine {1}:{2}".format(DOCKERS[docker_version], image, tag), + fout) + + # push! + output = run_cmd("{0} push {1}:{2}".format(DOCKERS[docker_version], + image, tag), + fout) + sha = _DIGEST_REGEX.search(output).group(1) + size = _SIZE_REGEX.search(output).group(1) + + # list + targets = notary_list(fout, image) + for target in targets: + if target[0] == docker_version: + assert_equality(target, [docker_version, sha, size, "targets"]) + return sha, size + + +def notary_list(fout, repo): + """ + Calls notary list on the repo and returns a list of lists of tags, shas, + sizes, and roles. + """ + clear_tuf() + output = run_cmd( + "{0} -d {1} list {2}".format(NOTARY_CLIENT, TRUST_DIR, repo), fout) + lines = output.strip().split("\n") + assert len(lines) >= 3, "not enough targets" + return [line.strip().split() for line in lines[2:]] + + +def test_pull_a(fout, docker_version, image, expected_tags): + """ + Pull -A on an image and ensure that all the expected tags are present + """ + clear_tuf() + # remove every image possible + for tag in expected_tags: + rmi(fout, docker_version, image, tag) + + # pull -a + output = run_cmd( + "{0} pull -a {1}".format(DOCKERS[docker_version], image), fout) + pulled_tags = _PULL_A_REGEX.findall(output) + + assert_equality(len(pulled_tags), len(expected_tags)) + for tag, info in expected_tags.iteritems(): + found = [pulled for pulled in pulled_tags if pulled[0] == tag] + assert found + assert_equality(found[0][1], info["sha"]) + + +def test_push(tempdir, docker_version, image, tag="", allow_push_failure=False, + do_after_first_push=None): + """ + Tests a push of an image by pushing with this docker version, and asserting + that all the other docker versions can pull it. + """ + if not tag: + tag = docker_version + + filename = os.path.join( + tempdir, "{0}_{1}_push_{2}").format(time(), docker_version, tag) + + with open(filename, 'wb') as fout: + try: + sha, size = push(fout, docker_version, image, tag=tag) + except subprocess.CalledProcessError: + if allow_push_failure: + return {"push": "failed"} + raise + + return_val = { + "push": { + "sha": sha, + "size": size + } + } + + if do_after_first_push is not None: + do_after_first_push(fout, image) + + for ver in DOCKERS: + try: + pull(fout, ver, image, docker_version, sha) + except subprocess.CalledProcessError: + return_val["push"][ver] = "pull failed" + else: + return_val["push"][ver] = "pull succeeded" + + return return_val + + +def test_docker_version(docker_version, repo_name="", do_after_first_push=None): + """ + Initialize a repo with one docker version. Test that all other docker + versions against that repo (both pulling and pushing). + """ + if not repo_name: + repo_name = "repo_by_{0}".format(docker_version) + tempdir = os.path.join(_TEMPDIR, repo_name) + os.makedirs(tempdir) + image = "{0}/{1}/{2}_{3}-{4}".format( + REGISTRY, REGISTRY_USERNAME, REPO_PREFIX, repo_name, time()) + + result = OrderedDict([ + (docker_version, test_push(tempdir, docker_version, image, + do_after_first_push=do_after_first_push)) + ]) + + # push again if we did something after the first push + if do_after_first_push: + tag = docker_version + "_push_again" + result[tag] = test_push( + tempdir, docker_version, image, tag=tag, allow_push_failure=True) + + for ver in DOCKERS: + if ver != docker_version: + result[ver] = test_push( + tempdir, ver, image, allow_push_failure=True) + + # find all the successfully pushed tags + expected_tags = {} + for ver in result: + if result[ver]["push"] != "failed": + expected_tags[ver] = result[ver]["push"] + + with open(os.path.join(tempdir, "pull_a"), 'wb') as fout: + for ver in DOCKERS: + try: + test_pull_a(fout, ver, image, expected_tags) + except subprocess.CalledProcessError: + result[ver]["pull-a"] = "failed" + else: + result[ver]["pull-a"] = "success" + + with open(os.path.join(tempdir, "notary_list"), 'wb') as fout: + targets = notary_list(fout, image) + assert_equality(len(targets), len(expected_tags)) + for tag, info in expected_tags.iteritems(): + found = [target for target in targets if target[0] == tag] + assert found + assert_equality( + found[0][1:], + [info["sha"], info["size"], "targets"]) + + result["list"] = "listed expected targets successfully" + + with open(os.path.join(tempdir, "result.json"), 'wb') as fout: + json.dump(result, fout, indent=2) + + return result + + +def rotate_to_server_snapshot(fout, image): + """ + Uses the notary client to rotate the snapshot key to be server-managed. + """ + run_cmd( + "{0} -d {1} key rotate {2} -t snapshot -r".format( + NOTARY_CLIENT, TRUST_DIR, image), + fout) + run_cmd( + "{0} -d {1} publish {2}".format(NOTARY_CLIENT, TRUST_DIR, image), + fout) + + +def test_all_docker_versions(): + """ + Initialize a repo with each docker version, and test that other docker + versions can read/write to it. + """ + print("Output files at", _TEMPDIR) + results = OrderedDict() + for docker_version in DOCKERS: + result = test_docker_version(docker_version) + print("\nRepo created with docker {0}:".format(docker_version)) + print(json.dumps(result, indent=2)) + results[docker_version] = result + + if docker_version != "1.10": + repo_name = "repo_by_{0}_snapshot_rotation".format(docker_version) + result = test_docker_version( + docker_version, repo_name=repo_name, + do_after_first_push=rotate_to_server_snapshot) + + print("\nRepo created with docker {0} and snapshot key rotated:" + .format(docker_version)) + print(json.dumps(result, indent=2)) + results[docker_version + "_snapshot_rotation"] = result + + + with open(os.path.join(_TEMPDIR, "total_results.json"), 'wb') as fout: + json.dump(results, fout, indent=2) + + print("\n\nFinal results:") + results["output_dir"] = _TEMPDIR + print(json.dumps(results, indent=2)) + + +if __name__ == "__main__": + setup() + test_all_docker_versions()