docker
/
docker-py
mirror of https://github.com/docker/docker-py.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2074 from docker/3.4.1-release 

3.4.1 release
This commit is contained in:
Joffrey F 2018-06-29 12:01:25 -07:00 committed by GitHub
parent f70545e89a bc28fd0ee2
commit 1e389b7cfe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 703 additions and 623 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docker/api/container.py
View File

@ -139,8 +139,9 @@ class ContainerApiMixin(object):
'changes': changes
}
u = self._url("/commit")
return self._result(self._post_json(u, data=conf, params=params),
json=True)
return self._result(
self._post_json(u, data=conf, params=params), json=True
)
def containers(self, quiet=False, all=False, trunc=False, latest=False,
since=None, before=None, limit=-1, size=False,

6
docker/auth.py
View File

@ -270,7 +270,7 @@ def load_config(config_path=None, config_dict=None):
"Couldn't find auth-related section ; attempting to interpret"
"as auth-only file"
)
return parse_auth(config_dict)
return {'auths': parse_auth(config_dict)}
def _load_legacy_config(config_file):
@ -287,14 +287,14 @@ def _load_legacy_config(config_file):
)
username, password = decode_auth(data[0])
return {
return {'auths': {
INDEX_NAME: {
'username': username,
'password': password,
'email': data[1],
'serveraddress': INDEX_URL,
}
}
}}
except Exception as e:
log.debug(e)
pass

214
docker/utils/build.py
View File

@ -1,13 +1,13 @@
import io
import os
import re
import six
import tarfile
import tempfile
import six
from .fnmatch import fnmatch
from ..constants import IS_WINDOWS_PLATFORM
from fnmatch import fnmatch
from itertools import chain
_SEP = re.compile('/|\\\\') if IS_WINDOWS_PLATFORM else re.compile('/')
@ -44,92 +44,9 @@ def exclude_paths(root, patterns, dockerfile=None):
if dockerfile is None:
dockerfile = 'Dockerfile'
def split_path(p):
return [pt for pt in re.split(_SEP, p) if pt and pt != '.']
def normalize(p):
# Leading and trailing slashes are not relevant. Yes,
# "foo.py/" must exclude the "foo.py" regular file. "."
# components are not relevant either, even if the whole
# pattern is only ".", as the Docker reference states: "For
# historical reasons, the pattern . is ignored."
# ".." component must be cleared with the potential previous
# component, regardless of whether it exists: "A preprocessing
# step [...] eliminates . and .. elements using Go's
# filepath.".
i = 0
split = split_path(p)
while i < len(split):
if split[i] == '..':
del split[i]
if i > 0:
del split[i - 1]
i -= 1
else:
i += 1
return split
patterns = (
(True, normalize(p[1:]))
if p.startswith('!') else
(False, normalize(p))
for p in patterns)
patterns = list(reversed(list(chain(
# Exclude empty patterns such as "." or the empty string.
filter(lambda p: p[1], patterns),
# Always include the Dockerfile and .dockerignore
[(True, split_path(dockerfile)), (True, ['.dockerignore'])]))))
return set(walk(root, patterns))
def walk(root, patterns, default=True):
"""
A collection of file lying below root that should be included according to
patterns.
"""
def match(p):
if p[1][0] == '**':
rec = (p[0], p[1][1:])
return [p] + (match(rec) if rec[1] else [rec])
elif fnmatch(f, p[1][0]):
return [(p[0], p[1][1:])]
else:
return []
for f in os.listdir(root):
cur = os.path.join(root, f)
# The patterns if recursing in that directory.
sub = list(chain(*(match(p) for p in patterns)))
# Whether this file is explicitely included / excluded.
hit = next((p[0] for p in sub if not p[1]), None)
# Whether this file is implicitely included / excluded.
matched = default if hit is None else hit
sub = list(filter(lambda p: p[1], sub))
if os.path.isdir(cur) and not os.path.islink(cur):
# Entirely skip directories if there are no chance any subfile will
# be included.
if all(not p[0] for p in sub) and not matched:
continue
# I think this would greatly speed up dockerignore handling by not
# recursing into directories we are sure would be entirely
# included, and only yielding the directory itself, which will be
# recursively archived anyway. However the current unit test expect
# the full list of subfiles and I'm not 100% sure it would make no
# difference yet.
# if all(p[0] for p in sub) and matched:
# yield f
# continue
children = False
for r in (os.path.join(f, p) for p in walk(cur, sub, matched)):
yield r
children = True
# The current unit tests expect directories only under those
# conditions. It might be simplifiable though.
if (not sub or not children) and hit or hit is None and default:
yield f
elif matched:
yield f
patterns.append('!' + dockerfile)
pm = PatternMatcher(patterns)
return set(pm.walk(root))
def build_file_list(root):
@ -217,3 +134,122 @@ def mkbuildcontext(dockerfile):
t.close()
f.seek(0)
return f
def split_path(p):
return [pt for pt in re.split(_SEP, p) if pt and pt != '.']
def normalize_slashes(p):
if IS_WINDOWS_PLATFORM:
return '/'.join(split_path(p))
return p
def walk(root, patterns, default=True):
pm = PatternMatcher(patterns)
return pm.walk(root)
# Heavily based on
# https://github.com/moby/moby/blob/master/pkg/fileutils/fileutils.go
class PatternMatcher(object):
def __init__(self, patterns):
self.patterns = list(filter(
lambda p: p.dirs, [Pattern(p) for p in patterns]
))
self.patterns.append(Pattern('!.dockerignore'))
def matches(self, filepath):
matched = False
parent_path = os.path.dirname(filepath)
parent_path_dirs = split_path(parent_path)
for pattern in self.patterns:
negative = pattern.exclusion
match = pattern.match(filepath)
if not match and parent_path != '':
if len(pattern.dirs) <= len(parent_path_dirs):
match = pattern.match(
os.path.sep.join(parent_path_dirs[:len(pattern.dirs)])
)
if match:
matched = not negative
return matched
def walk(self, root):
def rec_walk(current_dir):
for f in os.listdir(current_dir):
fpath = os.path.join(
os.path.relpath(current_dir, root), f
)
if fpath.startswith('.' + os.path.sep):
fpath = fpath[2:]
match = self.matches(fpath)
if not match:
yield fpath
cur = os.path.join(root, fpath)
if not os.path.isdir(cur) or os.path.islink(cur):
continue
if match:
# If we want to skip this file and it's a directory
# then we should first check to see if there's an
# excludes pattern (e.g. !dir/file) that starts with this
# dir. If so then we can't skip this dir.
skip = True
for pat in self.patterns:
if not pat.exclusion:
continue
if pat.cleaned_pattern.startswith(
normalize_slashes(fpath)):
skip = False
break
if skip:
continue
for sub in rec_walk(cur):
yield sub
return rec_walk(root)
class Pattern(object):
def __init__(self, pattern_str):
self.exclusion = False
if pattern_str.startswith('!'):
self.exclusion = True
pattern_str = pattern_str[1:]
self.dirs = self.normalize(pattern_str)
self.cleaned_pattern = '/'.join(self.dirs)
@classmethod
def normalize(cls, p):
# Leading and trailing slashes are not relevant. Yes,
# "foo.py/" must exclude the "foo.py" regular file. "."
# components are not relevant either, even if the whole
# pattern is only ".", as the Docker reference states: "For
# historical reasons, the pattern . is ignored."
# ".." component must be cleared with the potential previous
# component, regardless of whether it exists: "A preprocessing
# step [...] eliminates . and .. elements using Go's
# filepath.".
i = 0
split = split_path(p)
while i < len(split):
if split[i] == '..':
del split[i]
if i > 0:
del split[i - 1]
i -= 1
else:
i += 1
return split
def match(self, filepath):
return fnmatch(normalize_slashes(filepath), self.cleaned_pattern)

1
docker/utils/fnmatch.py
View File

@ -111,4 +111,5 @@ def translate(pat):
res = '%s[%s]' % (res, stuff)
else:
res = res + re.escape(c)
return res + '$'

2
docker/version.py
View File

@ -1,2 +1,2 @@
version = "3.4.0"
version = "3.4.1"
version_info = tuple([int(d) for d in version.split("-")[0].split(".")])

12
docs/change-log.md
View File

@ -1,6 +1,18 @@
Change log
==========
3.4.1
-----
[List of PRs / issues for this release](https://github.com/docker/docker-py/milestone/52?closed=1)
### Bugfixes
* Fixed a bug that caused auth values in config files written using one of the
legacy formats to be ignored
* Fixed issues with handling of double-wildcard `**` patterns in
`.dockerignore` files
3.4.0
-----

7
tests/helpers.py
View File

@ -123,7 +123,12 @@ def assert_cat_socket_detached_with_keys(sock, inputs):
sock.sendall(b'make sure the socket is closed\n')
else:
sock.sendall(b"make sure the socket is closed\n")
assert sock.recv(32) == b''
data = sock.recv(128)
# New in 18.06: error message is broadcast over the socket when reading
# after detach
assert data == b'' or data.startswith(
b'exec attach failed: error on attach stdin: read escape sequence'
)
def ctrl_with(char):

40
tests/integration/api_client_test.py
View File

@ -1,6 +1,3 @@
import base64
import os
import tempfile
import time
import unittest
import warnings
@ -24,43 +21,6 @@ class InformationTest(BaseAPIIntegrationTest):
assert 'Debug' in res
class LoadConfigTest(BaseAPIIntegrationTest):
def test_load_legacy_config(self):
folder = tempfile.mkdtemp()
self.tmp_folders.append(folder)
cfg_path = os.path.join(folder, '.dockercfg')
f = open(cfg_path, 'w')
auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii')
f.write('auth = {0}\n'.format(auth_))
f.write('email = sakuya@scarlet.net')
f.close()
cfg = docker.auth.load_config(cfg_path)
assert cfg[docker.auth.INDEX_NAME] is not None
cfg = cfg[docker.auth.INDEX_NAME]
assert cfg['username'] == 'sakuya'
assert cfg['password'] == 'izayoi'
assert cfg['email'] == 'sakuya@scarlet.net'
assert cfg.get('Auth') is None
def test_load_json_config(self):
folder = tempfile.mkdtemp()
self.tmp_folders.append(folder)
cfg_path = os.path.join(folder, '.dockercfg')
f = open(os.path.join(folder, '.dockercfg'), 'w')
auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii')
email_ = 'sakuya@scarlet.net'
f.write('{{"{0}": {{"auth": "{1}", "email": "{2}"}}}}\n'.format(
docker.auth.INDEX_URL, auth_, email_))
f.close()
cfg = docker.auth.load_config(cfg_path)
assert cfg[docker.auth.INDEX_URL] is not None
cfg = cfg[docker.auth.INDEX_URL]
assert cfg['username'] == 'sakuya'
assert cfg['password'] == 'izayoi'
assert cfg['email'] == 'sakuya@scarlet.net'
assert cfg.get('Auth') is None
class AutoDetectVersionTest(unittest.TestCase):
def test_client_init(self):
client = docker.APIClient(version='auto', **kwargs_from_env())

64
tests/unit/auth_test.py
View File

@ -282,22 +282,64 @@ class LoadConfigTest(unittest.TestCase):
cfg = auth.load_config(folder)
assert cfg is not None
def test_load_config(self):
def test_load_legacy_config(self):
folder = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, folder)
dockercfg_path = os.path.join(folder, '.dockercfg')
with open(dockercfg_path, 'w') as f:
cfg_path = os.path.join(folder, '.dockercfg')
auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii')
with open(cfg_path, 'w') as f:
f.write('auth = {0}\n'.format(auth_))
f.write('email = sakuya@scarlet.net')
cfg = auth.load_config(dockercfg_path)
assert auth.INDEX_NAME in cfg
assert cfg[auth.INDEX_NAME] is not None
cfg = cfg[auth.INDEX_NAME]
cfg = auth.load_config(cfg_path)
assert auth.resolve_authconfig(cfg) is not None
assert cfg['auths'][auth.INDEX_NAME] is not None
cfg = cfg['auths'][auth.INDEX_NAME]
assert cfg['username'] == 'sakuya'
assert cfg['password'] == 'izayoi'
assert cfg['email'] == 'sakuya@scarlet.net'
assert cfg.get('auth') is None
assert cfg.get('Auth') is None
def test_load_json_config(self):
folder = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, folder)
cfg_path = os.path.join(folder, '.dockercfg')
auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii')
email = 'sakuya@scarlet.net'
with open(cfg_path, 'w') as f:
json.dump(
{auth.INDEX_URL: {'auth': auth_, 'email': email}}, f
)
cfg = auth.load_config(cfg_path)
assert auth.resolve_authconfig(cfg) is not None
assert cfg['auths'][auth.INDEX_URL] is not None
cfg = cfg['auths'][auth.INDEX_URL]
assert cfg['username'] == 'sakuya'
assert cfg['password'] == 'izayoi'
assert cfg['email'] == email
assert cfg.get('Auth') is None
def test_load_modern_json_config(self):
folder = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, folder)
cfg_path = os.path.join(folder, 'config.json')
auth_ = base64.b64encode(b'sakuya:izayoi').decode('ascii')
email = 'sakuya@scarlet.net'
with open(cfg_path, 'w') as f:
json.dump({
'auths': {
auth.INDEX_URL: {
'auth': auth_, 'email': email
}
}
}, f)
cfg = auth.load_config(cfg_path)
assert auth.resolve_authconfig(cfg) is not None
assert cfg['auths'][auth.INDEX_URL] is not None
cfg = cfg['auths'][auth.INDEX_URL]
assert cfg['username'] == 'sakuya'
assert cfg['password'] == 'izayoi'
assert cfg['email'] == email
def test_load_config_with_random_name(self):
folder = tempfile.mkdtemp()
@ -318,7 +360,7 @@ class LoadConfigTest(unittest.TestCase):
with open(dockercfg_path, 'w') as f:
json.dump(config, f)
cfg = auth.load_config(dockercfg_path)
cfg = auth.load_config(dockercfg_path)['auths']
assert registry in cfg
assert cfg[registry] is not None
cfg = cfg[registry]
@ -345,7 +387,7 @@ class LoadConfigTest(unittest.TestCase):
json.dump(config, f)
with mock.patch.dict(os.environ, {'DOCKER_CONFIG': folder}):
cfg = auth.load_config(None)
cfg = auth.load_config(None)['auths']
assert registry in cfg
assert cfg[registry] is not None
cfg = cfg[registry]
@ -422,7 +464,7 @@ class LoadConfigTest(unittest.TestCase):
json.dump(config, f)
cfg = auth.load_config(dockercfg_path)
assert cfg == {}
assert cfg == {'auths': {}}
def test_load_config_invalid_auth_dict(self):
folder = tempfile.mkdtemp()

493
tests/unit/utils_build_test.py Normal file
View File

@ -0,0 +1,493 @@
# -*- coding: utf-8 -*-
import os
import os.path
import shutil
import socket
import tarfile
import tempfile
import unittest
from docker.constants import IS_WINDOWS_PLATFORM
from docker.utils import exclude_paths, tar
import pytest
from ..helpers import make_tree
def convert_paths(collection):
return set(map(convert_path, collection))
def convert_path(path):
return path.replace('/', os.path.sep)
class ExcludePathsTest(unittest.TestCase):
dirs = [
'foo',
'foo/bar',
'bar',
'target',
'target/subdir',
'subdir',
'subdir/target',
'subdir/target/subdir',
'subdir/subdir2',
'subdir/subdir2/target',
'subdir/subdir2/target/subdir'
]
files = [
'Dockerfile',
'Dockerfile.alt',
'.dockerignore',
'a.py',
'a.go',
'b.py',
'cde.py',
'foo/a.py',
'foo/b.py',
'foo/bar/a.py',
'bar/a.py',
'foo/Dockerfile3',
'target/file.txt',
'target/subdir/file.txt',
'subdir/file.txt',
'subdir/target/file.txt',
'subdir/target/subdir/file.txt',
'subdir/subdir2/file.txt',
'subdir/subdir2/target/file.txt',
'subdir/subdir2/target/subdir/file.txt',
]
all_paths = set(dirs + files)
def setUp(self):
self.base = make_tree(self.dirs, self.files)
def tearDown(self):
shutil.rmtree(self.base)
def exclude(self, patterns, dockerfile=None):
return set(exclude_paths(self.base, patterns, dockerfile=dockerfile))
def test_no_excludes(self):
assert self.exclude(['']) == convert_paths(self.all_paths)
def test_no_dupes(self):
paths = exclude_paths(self.base, ['!a.py'])
assert sorted(paths) == sorted(set(paths))
def test_wildcard_exclude(self):
assert self.exclude(['*']) == set(['Dockerfile', '.dockerignore'])
def test_exclude_dockerfile_dockerignore(self):
"""
Even if the .dockerignore file explicitly says to exclude
Dockerfile and/or .dockerignore, don't exclude them from
the actual tar file.
"""
assert self.exclude(['Dockerfile', '.dockerignore']) == convert_paths(
self.all_paths
)
def test_exclude_custom_dockerfile(self):
"""
If we're using a custom Dockerfile, make sure that's not
excluded.
"""
assert self.exclude(['*'], dockerfile='Dockerfile.alt') == set(
['Dockerfile.alt', '.dockerignore']
)
assert self.exclude(
['*'], dockerfile='foo/Dockerfile3'
) == convert_paths(set(['foo/Dockerfile3', '.dockerignore']))
# https://github.com/docker/docker-py/issues/1956
assert self.exclude(
['*'], dockerfile='./foo/Dockerfile3'
) == convert_paths(set(['foo/Dockerfile3', '.dockerignore']))
def test_exclude_dockerfile_child(self):
includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3')
assert convert_path('foo/Dockerfile3') in includes
assert convert_path('foo/a.py') not in includes
def test_single_filename(self):
assert self.exclude(['a.py']) == convert_paths(
self.all_paths - set(['a.py'])
)
def test_single_filename_leading_dot_slash(self):
assert self.exclude(['./a.py']) == convert_paths(
self.all_paths - set(['a.py'])
)
# As odd as it sounds, a filename pattern with a trailing slash on the
# end *will* result in that file being excluded.
def test_single_filename_trailing_slash(self):
assert self.exclude(['a.py/']) == convert_paths(
self.all_paths - set(['a.py'])
)
def test_wildcard_filename_start(self):
assert self.exclude(['*.py']) == convert_paths(
self.all_paths - set(['a.py', 'b.py', 'cde.py'])
)
def test_wildcard_with_exception(self):
assert self.exclude(['*.py', '!b.py']) == convert_paths(
self.all_paths - set(['a.py', 'cde.py'])
)
def test_wildcard_with_wildcard_exception(self):
assert self.exclude(['*.*', '!*.go']) == convert_paths(
self.all_paths - set([
'a.py', 'b.py', 'cde.py', 'Dockerfile.alt',
])
)
def test_wildcard_filename_end(self):
assert self.exclude(['a.*']) == convert_paths(
self.all_paths - set(['a.py', 'a.go'])
)
def test_question_mark(self):
assert self.exclude(['?.py']) == convert_paths(
self.all_paths - set(['a.py', 'b.py'])
)
def test_single_subdir_single_filename(self):
assert self.exclude(['foo/a.py']) == convert_paths(
self.all_paths - set(['foo/a.py'])
)
def test_single_subdir_single_filename_leading_slash(self):
assert self.exclude(['/foo/a.py']) == convert_paths(
self.all_paths - set(['foo/a.py'])
)
def test_exclude_include_absolute_path(self):
base = make_tree([], ['a.py', 'b.py'])
assert exclude_paths(
base,
['/*', '!/*.py']
) == set(['a.py', 'b.py'])
def test_single_subdir_with_path_traversal(self):
assert self.exclude(['foo/whoops/../a.py']) == convert_paths(
self.all_paths - set(['foo/a.py'])
)
def test_single_subdir_wildcard_filename(self):
assert self.exclude(['foo/*.py']) == convert_paths(
self.all_paths - set(['foo/a.py', 'foo/b.py'])
)
def test_wildcard_subdir_single_filename(self):
assert self.exclude(['*/a.py']) == convert_paths(
self.all_paths - set(['foo/a.py', 'bar/a.py'])
)
def test_wildcard_subdir_wildcard_filename(self):
assert self.exclude(['*/*.py']) == convert_paths(
self.all_paths - set(['foo/a.py', 'foo/b.py', 'bar/a.py'])
)
def test_directory(self):
assert self.exclude(['foo']) == convert_paths(
self.all_paths - set([
'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py',
'foo/Dockerfile3'
])
)
def test_directory_with_trailing_slash(self):
assert self.exclude(['foo']) == convert_paths(
self.all_paths - set([
'foo', 'foo/a.py', 'foo/b.py',
'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3'
])
)
def test_directory_with_single_exception(self):
assert self.exclude(['foo', '!foo/bar/a.py']) == convert_paths(
self.all_paths - set([
'foo/a.py', 'foo/b.py', 'foo', 'foo/bar',
'foo/Dockerfile3'
])
)
def test_directory_with_subdir_exception(self):
assert self.exclude(['foo', '!foo/bar']) == convert_paths(
self.all_paths - set([
'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3'
])
)
@pytest.mark.skipif(
not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows'
)
def test_directory_with_subdir_exception_win32_pathsep(self):
assert self.exclude(['foo', '!foo\\bar']) == convert_paths(
self.all_paths - set([
'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3'
])
)
def test_directory_with_wildcard_exception(self):
assert self.exclude(['foo', '!foo/*.py']) == convert_paths(
self.all_paths - set([
'foo/bar', 'foo/bar/a.py', 'foo', 'foo/Dockerfile3'
])
)
def test_subdirectory(self):
assert self.exclude(['foo/bar']) == convert_paths(
self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
)
@pytest.mark.skipif(
not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows'
)
def test_subdirectory_win32_pathsep(self):
assert self.exclude(['foo\\bar']) == convert_paths(
self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
)
def test_double_wildcard(self):
assert self.exclude(['**/a.py']) == convert_paths(
self.all_paths - set(
['a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py']
)
)
assert self.exclude(['foo/**/bar']) == convert_paths(
self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
)
def test_single_and_double_wildcard(self):
assert self.exclude(['**/target/*/*']) == convert_paths(
self.all_paths - set(
['target/subdir/file.txt',
'subdir/target/subdir/file.txt',
'subdir/subdir2/target/subdir/file.txt']
)
)
def test_trailing_double_wildcard(self):
assert self.exclude(['subdir/**']) == convert_paths(
self.all_paths - set(
['subdir/file.txt',
'subdir/target/file.txt',
'subdir/target/subdir/file.txt',
'subdir/subdir2/file.txt',
'subdir/subdir2/target/file.txt',
'subdir/subdir2/target/subdir/file.txt',
'subdir/target',
'subdir/target/subdir',
'subdir/subdir2',
'subdir/subdir2/target',
'subdir/subdir2/target/subdir']
)
)
def test_double_wildcard_with_exception(self):
assert self.exclude(['**', '!bar', '!foo/bar']) == convert_paths(
set([
'foo/bar', 'foo/bar/a.py', 'bar', 'bar/a.py', 'Dockerfile',
'.dockerignore',
])
)
def test_include_wildcard(self):
# This may be surprising but it matches the CLI's behavior
# (tested with 18.05.0-ce on linux)
base = make_tree(['a'], ['a/b.py'])
assert exclude_paths(
base,
['*', '!*/b.py']
) == set()
def test_last_line_precedence(self):
base = make_tree(
[],
['garbage.md',
'trash.md',
'README.md',
'README-bis.md',
'README-secret.md'])
assert exclude_paths(
base,
['*.md', '!README*.md', 'README-secret.md']
) == set(['README.md', 'README-bis.md'])
def test_parent_directory(self):
base = make_tree(
[],
['a.py',
'b.py',
'c.py'])
# Dockerignore reference stipulates that absolute paths are
# equivalent to relative paths, hence /../foo should be
# equivalent to ../foo. It also stipulates that paths are run
# through Go's filepath.Clean, which explicitely "replace
# "/.." by "/" at the beginning of a path".
assert exclude_paths(
base,
['../a.py', '/../b.py']
) == set(['c.py'])
class TarTest(unittest.TestCase):
def test_tar_with_excludes(self):
dirs = [
'foo',
'foo/bar',
'bar',
]
files = [
'Dockerfile',
'Dockerfile.alt',
'.dockerignore',
'a.py',
'a.go',
'b.py',
'cde.py',
'foo/a.py',
'foo/b.py',
'foo/bar/a.py',
'bar/a.py',
]
exclude = [
'*.py',
'!b.py',
'!a.go',
'foo',
'Dockerfile*',
'.dockerignore',
]
expected_names = set([
'Dockerfile',
'.dockerignore',
'a.go',
'b.py',
'bar',
'bar/a.py',
])
base = make_tree(dirs, files)
self.addCleanup(shutil.rmtree, base)
with tar(base, exclude=exclude) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == sorted(expected_names)
def test_tar_with_empty_directory(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'foo']
@pytest.mark.skipif(
IS_WINDOWS_PLATFORM or os.geteuid() == 0,
reason='root user always has access ; no chmod on Windows'
)
def test_tar_with_inaccessible_file(self):
base = tempfile.mkdtemp()
full_path = os.path.join(base, 'foo')
self.addCleanup(shutil.rmtree, base)
with open(full_path, 'w') as f:
f.write('content')
os.chmod(full_path, 0o222)
with pytest.raises(IOError) as ei:
tar(base)
assert 'Can not read file in context: {}'.format(full_path) in (
ei.exconly()
)
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
def test_tar_with_file_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
with open(os.path.join(base, 'foo'), 'w') as f:
f.write("content")
os.makedirs(os.path.join(base, 'bar'))
os.symlink('../foo', os.path.join(base, 'bar/foo'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
def test_tar_with_directory_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
os.symlink('../foo', os.path.join(base, 'bar/foo'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
def test_tar_with_broken_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
os.symlink('../baz', os.path.join(base, 'bar/foo'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32')
def test_tar_socket_file(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
sock = socket.socket(socket.AF_UNIX)
self.addCleanup(sock.close)
sock.bind(os.path.join(base, 'test.sock'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'foo']
def tar_test_negative_mtime_bug(self):
base = tempfile.mkdtemp()
filename = os.path.join(base, 'th.txt')
self.addCleanup(shutil.rmtree, base)
with open(filename, 'w') as f:
f.write('Invisible Full Moon')
os.utime(filename, (12345, -3600.0))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert tar_data.getnames() == ['th.txt']
assert tar_data.getmember('th.txt').mtime == -3600
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
def test_tar_directory_link(self):
dirs = ['a', 'b', 'a/c']
files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py']
base = make_tree(dirs, files)
self.addCleanup(shutil.rmtree, base)
os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
names = tar_data.getnames()
for member in dirs + files:
assert member in names
assert 'a/c/b' in names
assert 'a/c/b/utils.py' not in names

480
tests/unit/utils_test.py
View File

@ -5,29 +5,25 @@ import json
import os
import os.path
import shutil
import socket
import sys
import tarfile
import tempfile
import unittest
import pytest
import six
from docker.api.client import APIClient
from docker.constants import IS_WINDOWS_PLATFORM
from docker.errors import DockerException
from docker.utils import (
parse_repository_tag, parse_host, convert_filters, kwargs_from_env,
parse_bytes, parse_env_file, exclude_paths, convert_volume_binds,
decode_json_header, tar, split_command, parse_devices, update_headers,
convert_filters, convert_volume_binds, decode_json_header, kwargs_from_env,
parse_bytes, parse_devices, parse_env_file, parse_host,
parse_repository_tag, split_command, update_headers,
)
from docker.utils.ports import build_port_bindings, split_port
from docker.utils.utils import format_environment
from ..helpers import make_tree
import pytest
import six
TEST_CERT_DIR = os.path.join(
os.path.dirname(__file__),
@ -608,472 +604,6 @@ class PortsTest(unittest.TestCase):
assert port_bindings["2000"] == [("127.0.0.1", "2000")]
def convert_paths(collection):
return set(map(convert_path, collection))
def convert_path(path):
return path.replace('/', os.path.sep)
class ExcludePathsTest(unittest.TestCase):
dirs = [
'foo',
'foo/bar',
'bar',
'target',
'target/subdir',
'subdir',
'subdir/target',
'subdir/target/subdir',
'subdir/subdir2',
'subdir/subdir2/target',
'subdir/subdir2/target/subdir'
]
files = [
'Dockerfile',
'Dockerfile.alt',
'.dockerignore',
'a.py',
'a.go',
'b.py',
'cde.py',
'foo/a.py',
'foo/b.py',
'foo/bar/a.py',
'bar/a.py',
'foo/Dockerfile3',
'target/file.txt',
'target/subdir/file.txt',
'subdir/file.txt',
'subdir/target/file.txt',
'subdir/target/subdir/file.txt',
'subdir/subdir2/file.txt',
'subdir/subdir2/target/file.txt',
'subdir/subdir2/target/subdir/file.txt',
]
all_paths = set(dirs + files)
def setUp(self):
self.base = make_tree(self.dirs, self.files)
def tearDown(self):
shutil.rmtree(self.base)
def exclude(self, patterns, dockerfile=None):
return set(exclude_paths(self.base, patterns, dockerfile=dockerfile))
def test_no_excludes(self):
assert self.exclude(['']) == convert_paths(self.all_paths)
def test_no_dupes(self):
paths = exclude_paths(self.base, ['!a.py'])
assert sorted(paths) == sorted(set(paths))
def test_wildcard_exclude(self):
assert self.exclude(['*']) == set(['Dockerfile', '.dockerignore'])
def test_exclude_dockerfile_dockerignore(self):
"""
Even if the .dockerignore file explicitly says to exclude
Dockerfile and/or .dockerignore, don't exclude them from
the actual tar file.
"""
assert self.exclude(['Dockerfile', '.dockerignore']) == convert_paths(
self.all_paths
)
def test_exclude_custom_dockerfile(self):
"""
If we're using a custom Dockerfile, make sure that's not
excluded.
"""
assert self.exclude(['*'], dockerfile='Dockerfile.alt') == set(
['Dockerfile.alt', '.dockerignore']
)
assert self.exclude(
['*'], dockerfile='foo/Dockerfile3'
) == convert_paths(set(['foo/Dockerfile3', '.dockerignore']))
# https://github.com/docker/docker-py/issues/1956
assert self.exclude(
['*'], dockerfile='./foo/Dockerfile3'
) == convert_paths(set(['foo/Dockerfile3', '.dockerignore']))
def test_exclude_dockerfile_child(self):
includes = self.exclude(['foo/'], dockerfile='foo/Dockerfile3')
assert convert_path('foo/Dockerfile3') in includes
assert convert_path('foo/a.py') not in includes
def test_single_filename(self):
assert self.exclude(['a.py']) == convert_paths(
self.all_paths - set(['a.py'])
)
def test_single_filename_leading_dot_slash(self):
assert self.exclude(['./a.py']) == convert_paths(
self.all_paths - set(['a.py'])
)
# As odd as it sounds, a filename pattern with a trailing slash on the
# end *will* result in that file being excluded.
def test_single_filename_trailing_slash(self):
assert self.exclude(['a.py/']) == convert_paths(
self.all_paths - set(['a.py'])
)
def test_wildcard_filename_start(self):
assert self.exclude(['*.py']) == convert_paths(
self.all_paths - set(['a.py', 'b.py', 'cde.py'])
)
def test_wildcard_with_exception(self):
assert self.exclude(['*.py', '!b.py']) == convert_paths(
self.all_paths - set(['a.py', 'cde.py'])
)
def test_wildcard_with_wildcard_exception(self):
assert self.exclude(['*.*', '!*.go']) == convert_paths(
self.all_paths - set([
'a.py', 'b.py', 'cde.py', 'Dockerfile.alt',
])
)
def test_wildcard_filename_end(self):
assert self.exclude(['a.*']) == convert_paths(
self.all_paths - set(['a.py', 'a.go'])
)
def test_question_mark(self):
assert self.exclude(['?.py']) == convert_paths(
self.all_paths - set(['a.py', 'b.py'])
)
def test_single_subdir_single_filename(self):
assert self.exclude(['foo/a.py']) == convert_paths(
self.all_paths - set(['foo/a.py'])
)
def test_single_subdir_single_filename_leading_slash(self):
assert self.exclude(['/foo/a.py']) == convert_paths(
self.all_paths - set(['foo/a.py'])
)
def test_exclude_include_absolute_path(self):
base = make_tree([], ['a.py', 'b.py'])
assert exclude_paths(
base,
['/*', '!/*.py']
) == set(['a.py', 'b.py'])
def test_single_subdir_with_path_traversal(self):
assert self.exclude(['foo/whoops/../a.py']) == convert_paths(
self.all_paths - set(['foo/a.py'])
)
def test_single_subdir_wildcard_filename(self):
assert self.exclude(['foo/*.py']) == convert_paths(
self.all_paths - set(['foo/a.py', 'foo/b.py'])
)
def test_wildcard_subdir_single_filename(self):
assert self.exclude(['*/a.py']) == convert_paths(
self.all_paths - set(['foo/a.py', 'bar/a.py'])
)
def test_wildcard_subdir_wildcard_filename(self):
assert self.exclude(['*/*.py']) == convert_paths(
self.all_paths - set(['foo/a.py', 'foo/b.py', 'bar/a.py'])
)
def test_directory(self):
assert self.exclude(['foo']) == convert_paths(
self.all_paths - set([
'foo', 'foo/a.py', 'foo/b.py', 'foo/bar', 'foo/bar/a.py',
'foo/Dockerfile3'
])
)
def test_directory_with_trailing_slash(self):
assert self.exclude(['foo']) == convert_paths(
self.all_paths - set([
'foo', 'foo/a.py', 'foo/b.py',
'foo/bar', 'foo/bar/a.py', 'foo/Dockerfile3'
])
)
def test_directory_with_single_exception(self):
assert self.exclude(['foo', '!foo/bar/a.py']) == convert_paths(
self.all_paths - set([
'foo/a.py', 'foo/b.py', 'foo', 'foo/bar',
'foo/Dockerfile3'
])
)
def test_directory_with_subdir_exception(self):
assert self.exclude(['foo', '!foo/bar']) == convert_paths(
self.all_paths - set([
'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3'
])
)
@pytest.mark.skipif(
not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows'
)
def test_directory_with_subdir_exception_win32_pathsep(self):
assert self.exclude(['foo', '!foo\\bar']) == convert_paths(
self.all_paths - set([
'foo/a.py', 'foo/b.py', 'foo', 'foo/Dockerfile3'
])
)
def test_directory_with_wildcard_exception(self):
assert self.exclude(['foo', '!foo/*.py']) == convert_paths(
self.all_paths - set([
'foo/bar', 'foo/bar/a.py', 'foo', 'foo/Dockerfile3'
])
)
def test_subdirectory(self):
assert self.exclude(['foo/bar']) == convert_paths(
self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
)
@pytest.mark.skipif(
not IS_WINDOWS_PLATFORM, reason='Backslash patterns only on Windows'
)
def test_subdirectory_win32_pathsep(self):
assert self.exclude(['foo\\bar']) == convert_paths(
self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
)
def test_double_wildcard(self):
assert self.exclude(['**/a.py']) == convert_paths(
self.all_paths - set(
['a.py', 'foo/a.py', 'foo/bar/a.py', 'bar/a.py']
)
)
assert self.exclude(['foo/**/bar']) == convert_paths(
self.all_paths - set(['foo/bar', 'foo/bar/a.py'])
)
def test_single_and_double_wildcard(self):
assert self.exclude(['**/target/*/*']) == convert_paths(
self.all_paths - set(
['target/subdir/file.txt',
'subdir/target/subdir/file.txt',
'subdir/subdir2/target/subdir/file.txt']
)
)
def test_trailing_double_wildcard(self):
assert self.exclude(['subdir/**']) == convert_paths(
self.all_paths - set(
['subdir/file.txt',
'subdir/target/file.txt',
'subdir/target/subdir/file.txt',
'subdir/subdir2/file.txt',
'subdir/subdir2/target/file.txt',
'subdir/subdir2/target/subdir/file.txt',
'subdir/target',
'subdir/target/subdir',
'subdir/subdir2',
'subdir/subdir2/target',
'subdir/subdir2/target/subdir']
)
)
def test_include_wildcard(self):
base = make_tree(['a'], ['a/b.py'])
assert exclude_paths(
base,
['*', '!*/b.py']
) == convert_paths(['a/b.py'])
def test_last_line_precedence(self):
base = make_tree(
[],
['garbage.md',
'thrash.md',
'README.md',
'README-bis.md',
'README-secret.md'])
assert exclude_paths(
base,
['*.md', '!README*.md', 'README-secret.md']
) == set(['README.md', 'README-bis.md'])
def test_parent_directory(self):
base = make_tree(
[],
['a.py',
'b.py',
'c.py'])
# Dockerignore reference stipulates that absolute paths are
# equivalent to relative paths, hence /../foo should be
# equivalent to ../foo. It also stipulates that paths are run
# through Go's filepath.Clean, which explicitely "replace
# "/.." by "/" at the beginning of a path".
assert exclude_paths(
base,
['../a.py', '/../b.py']
) == set(['c.py'])
class TarTest(unittest.TestCase):
def test_tar_with_excludes(self):
dirs = [
'foo',
'foo/bar',
'bar',
]
files = [
'Dockerfile',
'Dockerfile.alt',
'.dockerignore',
'a.py',
'a.go',
'b.py',
'cde.py',
'foo/a.py',
'foo/b.py',
'foo/bar/a.py',
'bar/a.py',
]
exclude = [
'*.py',
'!b.py',
'!a.go',
'foo',
'Dockerfile*',
'.dockerignore',
]
expected_names = set([
'Dockerfile',
'.dockerignore',
'a.go',
'b.py',
'bar',
'bar/a.py',
])
base = make_tree(dirs, files)
self.addCleanup(shutil.rmtree, base)
with tar(base, exclude=exclude) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == sorted(expected_names)
def test_tar_with_empty_directory(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'foo']
@pytest.mark.skipif(
IS_WINDOWS_PLATFORM or os.geteuid() == 0,
reason='root user always has access ; no chmod on Windows'
)
def test_tar_with_inaccessible_file(self):
base = tempfile.mkdtemp()
full_path = os.path.join(base, 'foo')
self.addCleanup(shutil.rmtree, base)
with open(full_path, 'w') as f:
f.write('content')
os.chmod(full_path, 0o222)
with pytest.raises(IOError) as ei:
tar(base)
assert 'Can not read file in context: {}'.format(full_path) in (
ei.exconly()
)
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
def test_tar_with_file_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
with open(os.path.join(base, 'foo'), 'w') as f:
f.write("content")
os.makedirs(os.path.join(base, 'bar'))
os.symlink('../foo', os.path.join(base, 'bar/foo'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
def test_tar_with_directory_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
os.symlink('../foo', os.path.join(base, 'bar/foo'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
def test_tar_with_broken_symlinks(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
os.symlink('../baz', os.path.join(base, 'bar/foo'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'bar/foo', 'foo']
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No UNIX sockets on Win32')
def test_tar_socket_file(self):
base = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, base)
for d in ['foo', 'bar']:
os.makedirs(os.path.join(base, d))
sock = socket.socket(socket.AF_UNIX)
self.addCleanup(sock.close)
sock.bind(os.path.join(base, 'test.sock'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert sorted(tar_data.getnames()) == ['bar', 'foo']
def tar_test_negative_mtime_bug(self):
base = tempfile.mkdtemp()
filename = os.path.join(base, 'th.txt')
self.addCleanup(shutil.rmtree, base)
with open(filename, 'w') as f:
f.write('Invisible Full Moon')
os.utime(filename, (12345, -3600.0))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
assert tar_data.getnames() == ['th.txt']
assert tar_data.getmember('th.txt').mtime == -3600
@pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='No symlinks on Windows')
def test_tar_directory_link(self):
dirs = ['a', 'b', 'a/c']
files = ['a/hello.py', 'b/utils.py', 'a/c/descend.py']
base = make_tree(dirs, files)
self.addCleanup(shutil.rmtree, base)
os.symlink(os.path.join(base, 'b'), os.path.join(base, 'a/c/b'))
with tar(base) as archive:
tar_data = tarfile.open(fileobj=archive)
names = tar_data.getnames()
for member in dirs + files:
assert member in names
assert 'a/c/b' in names
assert 'a/c/b/utils.py' not in names
class FormatEnvironmentTest(unittest.TestCase):
def test_format_env_binary_unicode_value(self):
env_dict = {