From 427e3a60232a5b8ba6c2b91005908321366a0113 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 19 Jun 2015 02:12:32 +0200 Subject: [PATCH 01/43] Moved mem_limit and memswap_limit to host_config for API version >= 1.19 --- docker/client.py | 6 +++--- docker/utils/utils.py | 39 ++++++++++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/docker/client.py b/docker/client.py index 5781db69..294d0d3a 100644 --- a/docker/client.py +++ b/docker/client.py @@ -464,11 +464,11 @@ class Client(requests.Session): def create_container(self, image, command=None, hostname=None, user=None, detach=False, stdin_open=False, tty=False, - mem_limit=0, ports=None, environment=None, dns=None, - volumes=None, volumes_from=None, + mem_limit=None, ports=None, environment=None, + dns=None, volumes=None, volumes_from=None, network_disabled=False, name=None, entrypoint=None, cpu_shares=None, working_dir=None, domainname=None, - memswap_limit=0, cpuset=None, host_config=None, + memswap_limit=None, cpuset=None, host_config=None, mac_address=None, labels=None, volume_driver=None): if isinstance(volumes, six.string_types): diff --git a/docker/utils/utils.py b/docker/utils/utils.py index b322806c..4edbf11d 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -378,10 +378,21 @@ def create_host_config( dns=None, dns_search=None, volumes_from=None, network_mode=None, restart_policy=None, cap_add=None, cap_drop=None, devices=None, extra_hosts=None, read_only=None, pid_mode=None, ipc_mode=None, - security_opt=None, ulimits=None, log_config=None + security_opt=None, ulimits=None, log_config=None, mem_limit=None, + memswap_limit=None ): host_config = {} + if mem_limit is not None: + if isinstance(mem_limit, six.string_types): + mem_limit = parse_bytes(mem_limit) + host_config['Memory'] = mem_limit + + if memswap_limit is not None: + if isinstance(memswap_limit, six.string_types): + memswap_limit = parse_bytes(memswap_limit) + host_config['MemorySwap'] = memswap_limit + if pid_mode not in (None, 'host'): raise errors.DockerException( 'Invalid value for pid param: {0}'.format(pid_mode) @@ -498,10 +509,10 @@ def create_host_config( def create_container_config( version, image, command, hostname=None, user=None, detach=False, - stdin_open=False, tty=False, mem_limit=0, ports=None, environment=None, + stdin_open=False, tty=False, mem_limit=None, ports=None, environment=None, dns=None, volumes=None, volumes_from=None, network_disabled=False, entrypoint=None, cpu_shares=None, working_dir=None, domainname=None, - memswap_limit=0, cpuset=None, host_config=None, mac_address=None, + memswap_limit=None, cpuset=None, host_config=None, mac_address=None, labels=None, volume_driver=None ): if isinstance(command, six.string_types): @@ -517,10 +528,24 @@ def create_container_config( 'labels were only introduced in API version 1.18' ) - if volume_driver is not None and compare_version('1.19', version) < 0: - raise errors.InvalidVersion( - 'Volume drivers were only introduced in API version 1.19' - ) + if compare_version('1.19', version) < 0: + if volume_driver is not None: + raise errors.InvalidVersion( + 'Volume drivers were only introduced in API version 1.19' + ) + mem_limit = mem_limit if mem_limit is not None else 0 + memswap_limit = memswap_limit if memswap_limit is not None else 0 + else: + if mem_limit is not None: + raise errors.InvalidVersion( + 'mem_limit has been moved to host_config in API version 1.19' + ) + + if memswap_limit is not None: + raise errors.InvalidVersion( + 'memswap_limit has been moved to host_config in API ' + 'version 1.19' + ) if isinstance(labels, list): labels = dict((lbl, six.text_type('')) for lbl in labels) From a12ba1a96fb8a9855c55a2d51e29901467513fa5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 19 Jun 2015 02:13:12 +0200 Subject: [PATCH 02/43] Updated tests for mem_limit changes --- tests/test.py | 109 ++++++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 53 deletions(-) diff --git a/tests/test.py b/tests/test.py index bac95929..a7849b95 100644 --- a/tests/test.py +++ b/tests/test.py @@ -124,11 +124,10 @@ class DockerClientTest(Cleanup, base.BaseTestCase): if not cmd: cmd = ['true'] return {"Tty": False, "Image": img, "Cmd": cmd, - "AttachStdin": False, "Memory": 0, + "AttachStdin": False, "AttachStderr": True, "AttachStdout": True, "StdinOnce": False, "OpenStdin": False, "NetworkDisabled": False, - "MemorySwap": 0 } def test_ctor(self): @@ -337,11 +336,10 @@ class DockerClientTest(Cleanup, base.BaseTestCase): self.assertEqual(json.loads(args[1]['data']), json.loads(''' {"Tty": false, "Image": "busybox", "Cmd": ["true"], - "AttachStdin": false, "Memory": 0, + "AttachStdin": false, "AttachStderr": true, "AttachStdout": true, "StdinOnce": false, - "OpenStdin": false, "NetworkDisabled": false, - "MemorySwap": 0}''')) + "OpenStdin": false, "NetworkDisabled": false}''')) self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) @@ -361,12 +359,11 @@ class DockerClientTest(Cleanup, base.BaseTestCase): json.loads(''' {"Tty": false, "Image": "busybox", "Cmd": ["ls", "/mnt"], "AttachStdin": false, - "Volumes": {"/mnt": {}}, "Memory": 0, + "Volumes": {"/mnt": {}}, "AttachStderr": true, "AttachStdout": true, "OpenStdin": false, "StdinOnce": false, - "NetworkDisabled": false, - "MemorySwap": 0}''')) + "NetworkDisabled": false}''')) self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) @@ -386,12 +383,11 @@ class DockerClientTest(Cleanup, base.BaseTestCase): json.loads(''' {"Tty": false, "Image": "busybox", "Cmd": ["ls", "/mnt"], "AttachStdin": false, - "Volumes": {"/mnt": {}}, "Memory": 0, + "Volumes": {"/mnt": {}}, "AttachStderr": true, "AttachStdout": true, "OpenStdin": false, "StdinOnce": false, - "NetworkDisabled": false, - "MemorySwap": 0}''')) + "NetworkDisabled": false}''')) self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) @@ -409,7 +405,7 @@ class DockerClientTest(Cleanup, base.BaseTestCase): json.loads(''' {"Tty": false, "Image": "busybox", "Cmd": ["ls"], "AttachStdin": false, - "Memory": 0, "ExposedPorts": { + "ExposedPorts": { "1111/tcp": {}, "2222/udp": {}, "3333/tcp": {} @@ -417,8 +413,7 @@ class DockerClientTest(Cleanup, base.BaseTestCase): "AttachStderr": true, "AttachStdout": true, "OpenStdin": false, "StdinOnce": false, - "NetworkDisabled": false, - "MemorySwap": 0}''')) + "NetworkDisabled": false}''')) self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) @@ -436,13 +431,11 @@ class DockerClientTest(Cleanup, base.BaseTestCase): json.loads(''' {"Tty": false, "Image": "busybox", "Cmd": ["hello"], "AttachStdin": false, - "Memory": 0, "AttachStderr": true, "AttachStdout": true, "OpenStdin": false, "StdinOnce": false, "NetworkDisabled": false, - "Entrypoint": "cowsay", - "MemorySwap": 0}''')) + "Entrypoint": "cowsay"}''')) self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) @@ -460,13 +453,11 @@ class DockerClientTest(Cleanup, base.BaseTestCase): json.loads(''' {"Tty": false, "Image": "busybox", "Cmd": ["ls"], "AttachStdin": false, - "Memory": 0, "AttachStderr": true, "AttachStdout": true, "OpenStdin": false, "StdinOnce": false, "NetworkDisabled": false, - "CpuShares": 5, - "MemorySwap": 0}''')) + "CpuShares": 5}''')) self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) @@ -484,14 +475,12 @@ class DockerClientTest(Cleanup, base.BaseTestCase): json.loads(''' {"Tty": false, "Image": "busybox", "Cmd": ["ls"], "AttachStdin": false, - "Memory": 0, "AttachStderr": true, "AttachStdout": true, "OpenStdin": false, "StdinOnce": false, "NetworkDisabled": false, "Cpuset": "0,1", - "CpusetCpus": "0,1", - "MemorySwap": 0}''')) + "CpusetCpus": "0,1"}''')) self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) @@ -509,13 +498,11 @@ class DockerClientTest(Cleanup, base.BaseTestCase): json.loads(''' {"Tty": false, "Image": "busybox", "Cmd": ["ls"], "AttachStdin": false, - "Memory": 0, "AttachStderr": true, "AttachStdout": true, "OpenStdin": false, "StdinOnce": false, "NetworkDisabled": false, - "WorkingDir": "/root", - "MemorySwap": 0}''')) + "WorkingDir": "/root"}''')) self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) @@ -531,11 +518,10 @@ class DockerClientTest(Cleanup, base.BaseTestCase): self.assertEqual(json.loads(args[1]['data']), json.loads(''' {"Tty": false, "Image": "busybox", "Cmd": ["true"], - "AttachStdin": true, "Memory": 0, + "AttachStdin": true, "AttachStderr": true, "AttachStdout": true, "StdinOnce": true, - "OpenStdin": true, "NetworkDisabled": false, - "MemorySwap": 0}''')) + "OpenStdin": true, "NetworkDisabled": false}''')) self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) @@ -581,78 +567,95 @@ class DockerClientTest(Cleanup, base.BaseTestCase): self.assertEqual(json.loads(args[1]['data']), json.loads(''' {"Tty": false, "Image": "busybox", "Cmd": ["true"], - "AttachStdin": false, "Memory": 0, + "AttachStdin": false, "AttachStderr": true, "AttachStdout": true, "StdinOnce": false, - "OpenStdin": false, "NetworkDisabled": false, - "MemorySwap": 0}''')) + "OpenStdin": false, "NetworkDisabled": false}''')) self.assertEqual(args[1]['headers'], {'Content-Type': 'application/json'}) self.assertEqual(args[1]['params'], {'name': 'marisa-kirisame'}) def test_create_container_with_mem_limit_as_int(self): try: - self.client.create_container('busybox', 'true', - mem_limit=128.0) + self.client.create_container( + 'busybox', 'true', host_config=create_host_config( + mem_limit=128.0 + ) + ) except Exception as e: self.fail('Command should not raise exception: {0}'.format(e)) args = fake_request.call_args data = json.loads(args[1]['data']) - self.assertEqual(data['Memory'], 128.0) + self.assertEqual(data['HostConfig']['Memory'], 128.0) def test_create_container_with_mem_limit_as_string(self): try: - self.client.create_container('busybox', 'true', - mem_limit='128') + self.client.create_container( + 'busybox', 'true', host_config=create_host_config( + mem_limit='128' + ) + ) except Exception as e: self.fail('Command should not raise exception: {0}'.format(e)) args = fake_request.call_args data = json.loads(args[1]['data']) - self.assertEqual(data['Memory'], 128.0) + self.assertEqual(data['HostConfig']['Memory'], 128.0) def test_create_container_with_mem_limit_as_string_with_k_unit(self): try: - self.client.create_container('busybox', 'true', - mem_limit='128k') + self.client.create_container( + 'busybox', 'true', host_config=create_host_config( + mem_limit='128k' + ) + ) except Exception as e: self.fail('Command should not raise exception: {0}'.format(e)) args = fake_request.call_args data = json.loads(args[1]['data']) - self.assertEqual(data['Memory'], 128.0 * 1024) + self.assertEqual(data['HostConfig']['Memory'], 128.0 * 1024) def test_create_container_with_mem_limit_as_string_with_m_unit(self): try: - self.client.create_container('busybox', 'true', - mem_limit='128m') + self.client.create_container( + 'busybox', 'true', host_config=create_host_config( + mem_limit='128m' + ) + ) + except Exception as e: self.fail('Command should not raise exception: {0}'.format(e)) args = fake_request.call_args data = json.loads(args[1]['data']) - self.assertEqual(data['Memory'], 128.0 * 1024 * 1024) + self.assertEqual(data['HostConfig']['Memory'], 128.0 * 1024 * 1024) def test_create_container_with_mem_limit_as_string_with_g_unit(self): try: - self.client.create_container('busybox', 'true', - mem_limit='128g') + self.client.create_container( + 'busybox', 'true', host_config=create_host_config( + mem_limit='128g' + ) + ) except Exception as e: self.fail('Command should not raise exception: {0}'.format(e)) args = fake_request.call_args data = json.loads(args[1]['data']) - self.assertEqual(data['Memory'], 128.0 * 1024 * 1024 * 1024) + self.assertEqual( + data['HostConfig']['Memory'], 128.0 * 1024 * 1024 * 1024 + ) def test_create_container_with_mem_limit_as_string_with_wrong_value(self): - self.assertRaises(docker.errors.DockerException, - self.client.create_container, - 'busybox', 'true', mem_limit='128p') + self.assertRaises( + docker.errors.DockerException, create_host_config, mem_limit='128p' + ) - self.assertRaises(docker.errors.DockerException, - self.client.create_container, - 'busybox', 'true', mem_limit='1f28') + self.assertRaises( + docker.errors.DockerException, create_host_config, mem_limit='1f28' + ) def test_start_container(self): try: From e1329092445610f5918fe2dd29ed673b51abad19 Mon Sep 17 00:00:00 2001 From: Anthony Baire Date: Mon, 29 Jun 2015 20:05:01 +0200 Subject: [PATCH 03/43] fix premature EOF detection in streams The docker engine may send empty chunks of data in the stream (especially since https://github.com/docker/docker/pull/13033) They should not be taken for EOF Signed-off-by: Anthony Baire --- docker/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/client.py b/docker/client.py index 8bcc419e..8748a4f7 100644 --- a/docker/client.py +++ b/docker/client.py @@ -233,7 +233,7 @@ class Client(requests.Session): break _, length = struct.unpack('>BxxxL', header) if not length: - break + continue data = response.raw.read(length) if not data: break From b95948711785374a8f8446acb91156b00d0dbc48 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 30 Jun 2015 03:00:10 +0200 Subject: [PATCH 04/43] Added git@ as a valid prefix for remote build paths --- docker/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/client.py b/docker/client.py index 998ebacb..0a932482 100644 --- a/docker/client.py +++ b/docker/client.py @@ -317,7 +317,7 @@ class Client(requests.Session): elif fileobj is not None: context = utils.mkbuildcontext(fileobj) elif path.startswith(('http://', 'https://', - 'git://', 'github.com/')): + 'git://', 'github.com/', 'git@')): remote = path elif not os.path.isdir(path): raise TypeError("You must specify a directory to build in path") From 9e87884ba8cbddc0c7f2ad7ccdc11e172e844ac6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 24 Jun 2015 22:29:11 +0200 Subject: [PATCH 05/43] Fix Unix socket adapter bug with double slash in path + regression test --- docker/client.py | 15 ++++++++++++--- tests/integration_test.py | 9 +++++++-- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docker/client.py b/docker/client.py index 0a932482..fdd4fc55 100644 --- a/docker/client.py +++ b/docker/client.py @@ -52,15 +52,16 @@ class Client(requests.Session): base_url = utils.parse_host(base_url) if base_url.startswith('http+unix://'): - unix_socket_adapter = unixconn.UnixAdapter(base_url, timeout) - self.mount('http+docker://', unix_socket_adapter) + self._adapter = unixconn.UnixAdapter(base_url, timeout) + self.mount('http+docker://', self._adapter) self.base_url = 'http+docker://localunixsocket' else: # Use SSLAdapter for the ability to specify SSL version if isinstance(tls, TLSConfig): tls.configure_client(self) elif tls: - self.mount('https://', ssladapter.SSLAdapter()) + self._adapter = ssladapter.SSLAdapter() + self.mount('https://', self._adapter) self.base_url = base_url # version detection needs to be after unix adapter mounting @@ -243,6 +244,14 @@ class Client(requests.Session): def api_version(self): return self._version + def get_adapter(self, url): + try: + return super(Client, self).get_adapter(url) + except requests.exceptions.InvalidSchema as e: + if self._adapter: + return self._adapter + raise e + @check_resource def attach(self, container, stdout=True, stderr=True, stream=False, logs=False): diff --git a/tests/integration_test.py b/tests/integration_test.py index 4b9869e2..ac4a8717 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -872,8 +872,8 @@ class TestRunContainerStreaming(BaseTestCase): id = container['Id'] self.client.start(id) self.tmp_containers.append(id) - socket = self.client.attach_socket(container, ws=False) - self.assertTrue(socket.fileno() > -1) + sock = self.client.attach_socket(container, ws=False) + self.assertTrue(sock.fileno() > -1) class TestPauseUnpauseContainer(BaseTestCase): @@ -1467,12 +1467,17 @@ class TestRegressions(BaseTestCase): result = self.client.containers(all=True, trunc=True) self.assertEqual(len(result[0]['Id']), 12) + def test_647(self): + with self.assertRaises(docker.errors.APIError): + self.client.inspect_image('gensokyo.jp//kirisame') + def test_649(self): self.client.timeout = None ctnr = self.client.create_container('busybox', ['sleep', '2']) self.client.start(ctnr) self.client.stop(ctnr) + if __name__ == '__main__': c = docker.Client(base_url=DEFAULT_BASE_URL) c.pull('busybox') From d66369c29c35d61e63851ecf47018df2b48aa30d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 24 Jun 2015 22:29:11 +0200 Subject: [PATCH 06/43] Fix adapter bug + regression test --- tests/integration_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration_test.py b/tests/integration_test.py index ac4a8717..2a639e2c 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -1477,7 +1477,6 @@ class TestRegressions(BaseTestCase): self.client.start(ctnr) self.client.stop(ctnr) - if __name__ == '__main__': c = docker.Client(base_url=DEFAULT_BASE_URL) c.pull('busybox') From 4f89ca73da90f4dc30b5b278f474bf5850e75488 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 24 Jun 2015 22:49:16 +0200 Subject: [PATCH 07/43] ClientBase class to extract utility methods and constructor and sanitize Client class --- docker/client.py | 232 +------------------------------------ docker/clientbase.py | 235 ++++++++++++++++++++++++++++++++++++++ tests/integration_test.py | 1 + 3 files changed, 241 insertions(+), 227 deletions(-) create mode 100644 docker/clientbase.py diff --git a/docker/client.py b/docker/client.py index 321989f5..bb12e000 100644 --- a/docker/client.py +++ b/docker/client.py @@ -12,246 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import os import re import shlex -import struct import warnings from datetime import datetime -import requests -import requests.exceptions import six -import websocket - +from . import clientbase from . import constants from . import errors from .auth import auth -from .unixconn import unixconn -from .ssladapter import ssladapter from .utils import utils, check_resource -from .tls import TLSConfig -class Client(requests.Session): - def __init__(self, base_url=None, version=None, - timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False): - super(Client, self).__init__() - - if tls and not base_url.startswith('https://'): - raise errors.TLSParameterError( - 'If using TLS, the base_url argument must begin with ' - '"https://".') - - self.base_url = base_url - self.timeout = timeout - - self._auth_configs = auth.load_config() - - base_url = utils.parse_host(base_url) - if base_url.startswith('http+unix://'): - self._adapter = unixconn.UnixAdapter(base_url, timeout) - self.mount('http+docker://', self._adapter) - self.base_url = 'http+docker://localunixsocket' - else: - # Use SSLAdapter for the ability to specify SSL version - if isinstance(tls, TLSConfig): - tls.configure_client(self) - elif tls: - self._adapter = ssladapter.SSLAdapter() - self.mount('https://', self._adapter) - self.base_url = base_url - - # version detection needs to be after unix adapter mounting - if version is None: - self._version = constants.DEFAULT_DOCKER_API_VERSION - elif isinstance(version, six.string_types): - if version.lower() == 'auto': - self._version = self._retrieve_server_version() - else: - self._version = version - else: - raise errors.DockerException( - 'Version parameter must be a string or None. Found {0}'.format( - type(version).__name__ - ) - ) - - def _retrieve_server_version(self): - try: - return self.version(api_version=False)["ApiVersion"] - except KeyError: - raise errors.DockerException( - 'Invalid response from docker daemon: key "ApiVersion"' - ' is missing.' - ) - except Exception as e: - raise errors.DockerException( - 'Error while fetching server API version: {0}'.format(e) - ) - - def _set_request_timeout(self, kwargs): - """Prepare the kwargs for an HTTP request by inserting the timeout - parameter, if not already present.""" - kwargs.setdefault('timeout', self.timeout) - return kwargs - - def _post(self, url, **kwargs): - return self.post(url, **self._set_request_timeout(kwargs)) - - def _get(self, url, **kwargs): - return self.get(url, **self._set_request_timeout(kwargs)) - - def _delete(self, url, **kwargs): - return self.delete(url, **self._set_request_timeout(kwargs)) - - def _url(self, path, versioned_api=True): - if versioned_api: - return '{0}/v{1}{2}'.format(self.base_url, self._version, path) - else: - return '{0}{1}'.format(self.base_url, path) - - def _raise_for_status(self, response, explanation=None): - """Raises stored :class:`APIError`, if one occurred.""" - try: - response.raise_for_status() - except requests.exceptions.HTTPError as e: - raise errors.APIError(e, response, explanation=explanation) - - def _result(self, response, json=False, binary=False): - assert not (json and binary) - self._raise_for_status(response) - - if json: - return response.json() - if binary: - return response.content - return response.text - - def _post_json(self, url, data, **kwargs): - # Go <1.1 can't unserialize null to a string - # so we do this disgusting thing here. - data2 = {} - if data is not None: - for k, v in six.iteritems(data): - if v is not None: - data2[k] = v - - if 'headers' not in kwargs: - kwargs['headers'] = {} - kwargs['headers']['Content-Type'] = 'application/json' - return self._post(url, data=json.dumps(data2), **kwargs) - - def _attach_params(self, override=None): - return override or { - 'stdout': 1, - 'stderr': 1, - 'stream': 1 - } - - @check_resource - def _attach_websocket(self, container, params=None): - url = self._url("/containers/{0}/attach/ws".format(container)) - req = requests.Request("POST", url, params=self._attach_params(params)) - full_url = req.prepare().url - full_url = full_url.replace("http://", "ws://", 1) - full_url = full_url.replace("https://", "wss://", 1) - return self._create_websocket_connection(full_url) - - def _create_websocket_connection(self, url): - return websocket.create_connection(url) - - def _get_raw_response_socket(self, response): - self._raise_for_status(response) - if six.PY3: - sock = response.raw._fp.fp.raw - else: - sock = response.raw._fp.fp._sock - try: - # Keep a reference to the response to stop it being garbage - # collected. If the response is garbage collected, it will - # close TLS sockets. - sock._response = response - except AttributeError: - # UNIX sockets can't have attributes set on them, but that's - # fine because we won't be doing TLS over them - pass - - return sock - - def _stream_helper(self, response, decode=False): - """Generator for data coming from a chunked-encoded HTTP response.""" - if response.raw._fp.chunked: - reader = response.raw - while not reader.closed: - # this read call will block until we get a chunk - data = reader.read(1) - if not data: - break - if reader._fp.chunk_left: - data += reader.read(reader._fp.chunk_left) - if decode: - if six.PY3: - data = data.decode('utf-8') - data = json.loads(data) - yield data - else: - # Response isn't chunked, meaning we probably - # encountered an error immediately - yield self._result(response) - - def _multiplexed_buffer_helper(self, response): - """A generator of multiplexed data blocks read from a buffered - response.""" - buf = self._result(response, binary=True) - walker = 0 - while True: - if len(buf[walker:]) < 8: - break - _, length = struct.unpack_from('>BxxxL', buf[walker:]) - start = walker + constants.STREAM_HEADER_SIZE_BYTES - end = start + length - walker = end - yield buf[start:end] - - def _multiplexed_response_stream_helper(self, response): - """A generator of multiplexed data blocks coming from a response - stream.""" - - # Disable timeout on the underlying socket to prevent - # Read timed out(s) for long running processes - socket = self._get_raw_response_socket(response) - if six.PY3: - socket._sock.settimeout(None) - else: - socket.settimeout(None) - - while True: - header = response.raw.read(constants.STREAM_HEADER_SIZE_BYTES) - if not header: - break - _, length = struct.unpack('>BxxxL', header) - if not length: - break - data = response.raw.read(length) - if not data: - break - yield data - - @property - def api_version(self): - return self._version - - def get_adapter(self, url): - try: - return super(Client, self).get_adapter(url) - except requests.exceptions.InvalidSchema as e: - if self._adapter: - return self._adapter - raise e - +class Client(clientbase.ClientBase): @check_resource def attach(self, container, stdout=True, stderr=True, stream=False, logs=False): @@ -745,7 +521,9 @@ class Client(requests.Session): @check_resource def inspect_image(self, image): return self._result( - self._get(self._url("/images/{0}/json".format(image))), + self._get( + self._url("/images/{0}/json".format(image.replace('/', '%2F'))) + ), True ) diff --git a/docker/clientbase.py b/docker/clientbase.py new file mode 100644 index 00000000..e51bf3ec --- /dev/null +++ b/docker/clientbase.py @@ -0,0 +1,235 @@ +import json +import struct + +import requests +import requests.exceptions +import six +import websocket + + +from . import constants +from . import errors +from .auth import auth +from .unixconn import unixconn +from .ssladapter import ssladapter +from .utils import utils, check_resource +from .tls import TLSConfig + + +class ClientBase(requests.Session): + def __init__(self, base_url=None, version=None, + timeout=constants.DEFAULT_TIMEOUT_SECONDS, tls=False): + super(ClientBase, self).__init__() + + if tls and not base_url.startswith('https://'): + raise errors.TLSParameterError( + 'If using TLS, the base_url argument must begin with ' + '"https://".') + + self.base_url = base_url + self.timeout = timeout + + self._auth_configs = auth.load_config() + + base_url = utils.parse_host(base_url) + if base_url.startswith('http+unix://'): + self._custom_adapter = unixconn.UnixAdapter(base_url, timeout) + self.mount('http+docker://', self._custom_adapter) + self.base_url = 'http+docker://localunixsocket' + else: + # Use SSLAdapter for the ability to specify SSL version + if isinstance(tls, TLSConfig): + tls.configure_client(self) + elif tls: + self._custom_adapter = ssladapter.SSLAdapter() + self.mount('https://', self._custom_adapter) + self.base_url = base_url + + # version detection needs to be after unix adapter mounting + if version is None: + self._version = constants.DEFAULT_DOCKER_API_VERSION + elif isinstance(version, six.string_types): + if version.lower() == 'auto': + self._version = self._retrieve_server_version() + else: + self._version = version + else: + raise errors.DockerException( + 'Version parameter must be a string or None. Found {0}'.format( + type(version).__name__ + ) + ) + + def _retrieve_server_version(self): + try: + return self.version(api_version=False)["ApiVersion"] + except KeyError: + raise errors.DockerException( + 'Invalid response from docker daemon: key "ApiVersion"' + ' is missing.' + ) + except Exception as e: + raise errors.DockerException( + 'Error while fetching server API version: {0}'.format(e) + ) + + def _set_request_timeout(self, kwargs): + """Prepare the kwargs for an HTTP request by inserting the timeout + parameter, if not already present.""" + kwargs.setdefault('timeout', self.timeout) + return kwargs + + def _post(self, url, **kwargs): + return self.post(url, **self._set_request_timeout(kwargs)) + + def _get(self, url, **kwargs): + return self.get(url, **self._set_request_timeout(kwargs)) + + def _delete(self, url, **kwargs): + return self.delete(url, **self._set_request_timeout(kwargs)) + + def _url(self, path, versioned_api=True): + if versioned_api: + return '{0}/v{1}{2}'.format(self.base_url, self._version, path) + else: + return '{0}{1}'.format(self.base_url, path) + + def _raise_for_status(self, response, explanation=None): + """Raises stored :class:`APIError`, if one occurred.""" + try: + response.raise_for_status() + except requests.exceptions.HTTPError as e: + raise errors.APIError(e, response, explanation=explanation) + + def _result(self, response, json=False, binary=False): + assert not (json and binary) + self._raise_for_status(response) + + if json: + return response.json() + if binary: + return response.content + return response.text + + def _post_json(self, url, data, **kwargs): + # Go <1.1 can't unserialize null to a string + # so we do this disgusting thing here. + data2 = {} + if data is not None: + for k, v in six.iteritems(data): + if v is not None: + data2[k] = v + + if 'headers' not in kwargs: + kwargs['headers'] = {} + kwargs['headers']['Content-Type'] = 'application/json' + return self._post(url, data=json.dumps(data2), **kwargs) + + def _attach_params(self, override=None): + return override or { + 'stdout': 1, + 'stderr': 1, + 'stream': 1 + } + + @check_resource + def _attach_websocket(self, container, params=None): + url = self._url("/containers/{0}/attach/ws".format(container)) + req = requests.Request("POST", url, params=self._attach_params(params)) + full_url = req.prepare().url + full_url = full_url.replace("http://", "ws://", 1) + full_url = full_url.replace("https://", "wss://", 1) + return self._create_websocket_connection(full_url) + + def _create_websocket_connection(self, url): + return websocket.create_connection(url) + + def _get_raw_response_socket(self, response): + self._raise_for_status(response) + if six.PY3: + sock = response.raw._fp.fp.raw + else: + sock = response.raw._fp.fp._sock + try: + # Keep a reference to the response to stop it being garbage + # collected. If the response is garbage collected, it will + # close TLS sockets. + sock._response = response + except AttributeError: + # UNIX sockets can't have attributes set on them, but that's + # fine because we won't be doing TLS over them + pass + + return sock + + def _stream_helper(self, response, decode=False): + """Generator for data coming from a chunked-encoded HTTP response.""" + if response.raw._fp.chunked: + reader = response.raw + while not reader.closed: + # this read call will block until we get a chunk + data = reader.read(1) + if not data: + break + if reader._fp.chunk_left: + data += reader.read(reader._fp.chunk_left) + if decode: + if six.PY3: + data = data.decode('utf-8') + data = json.loads(data) + yield data + else: + # Response isn't chunked, meaning we probably + # encountered an error immediately + yield self._result(response) + + def _multiplexed_buffer_helper(self, response): + """A generator of multiplexed data blocks read from a buffered + response.""" + buf = self._result(response, binary=True) + walker = 0 + while True: + if len(buf[walker:]) < 8: + break + _, length = struct.unpack_from('>BxxxL', buf[walker:]) + start = walker + constants.STREAM_HEADER_SIZE_BYTES + end = start + length + walker = end + yield buf[start:end] + + def _multiplexed_response_stream_helper(self, response): + """A generator of multiplexed data blocks coming from a response + stream.""" + + # Disable timeout on the underlying socket to prevent + # Read timed out(s) for long running processes + socket = self._get_raw_response_socket(response) + if six.PY3: + socket._sock.settimeout(None) + else: + socket.settimeout(None) + + while True: + header = response.raw.read(constants.STREAM_HEADER_SIZE_BYTES) + if not header: + break + _, length = struct.unpack('>BxxxL', header) + if not length: + break + data = response.raw.read(length) + if not data: + break + yield data + + def get_adapter(self, url): + try: + return super(ClientBase, self).get_adapter(url) + except requests.exceptions.InvalidSchema as e: + if self._custom_adapter: + return self._custom_adapter + else: + raise e + + @property + def api_version(self): + return self._version diff --git a/tests/integration_test.py b/tests/integration_test.py index 2a639e2c..ac4a8717 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -1477,6 +1477,7 @@ class TestRegressions(BaseTestCase): self.client.start(ctnr) self.client.stop(ctnr) + if __name__ == '__main__': c = docker.Client(base_url=DEFAULT_BASE_URL) c.pull('busybox') From 3bc9f52d9dc9e176e7435ff94a1d21b5e9bd194d Mon Sep 17 00:00:00 2001 From: Peter Kowalczyk Date: Thu, 2 Jul 2015 10:42:08 -0500 Subject: [PATCH 08/43] Fix missing apostrophes in docs --- docs/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index 4b641471..5a3b3222 100644 --- a/docs/api.md +++ b/docs/api.md @@ -184,7 +184,7 @@ information on how to create port bindings and volume mappings. The `mem_limit` variable accepts float values (which represent the memory limit of the created container in bytes) or a string with a units identification char -('100000b', 1000k', 128m', '1g'). If a string is specified without a units +('100000b', '1000k', '128m', '1g'). If a string is specified without a units character, bytes are assumed as an intended unit. `volumes_from` and `dns` arguments raise [TypeError]( From 70b921f8a399c83c51ca363cea685e6003abcdaf Mon Sep 17 00:00:00 2001 From: Dan O'Reilly Date: Mon, 6 Jul 2015 18:20:41 -0400 Subject: [PATCH 09/43] Fix handling output from tty-enabled containers. Treat output from TTY-enabled containers as raw streams, rather than as multiplexed streams. The docker API docs specify that tty-enabled containers don't multiplex. Also update tests to pass with these changes, and changed the code used to read raw streams to not read line-by-line, and to not skip empty lines. Addresses issue #630 Signed-off-by: Dan O'Reilly --- docker/client.py | 46 +++----------------------------------------- docker/clientbase.py | 40 ++++++++++++++++++++++++++++++++++++++ tests/fake_api.py | 4 ++-- tests/test.py | 46 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 87 insertions(+), 49 deletions(-) diff --git a/docker/client.py b/docker/client.py index bb12e000..17b7da10 100644 --- a/docker/client.py +++ b/docker/client.py @@ -40,28 +40,7 @@ class Client(clientbase.ClientBase): u = self._url("/containers/{0}/attach".format(container)) response = self._post(u, params=params, stream=stream) - # Stream multi-plexing was only introduced in API v1.6. Anything before - # that needs old-style streaming. - if utils.compare_version('1.6', self._version) < 0: - def stream_result(): - self._raise_for_status(response) - for line in response.iter_lines(chunk_size=1, - decode_unicode=True): - # filter out keep-alive new lines - if line: - yield line - - return stream_result() if stream else \ - self._result(response, binary=True) - - sep = bytes() if six.PY3 else str() - - if stream: - return self._multiplexed_response_stream_helper(response) - else: - return sep.join( - [x for x in self._multiplexed_buffer_helper(response)] - ) + return self._get_result(container, stream, response) @check_resource def attach_socket(self, container, params=None, ws=False): @@ -363,17 +342,7 @@ class Client(clientbase.ClientBase): res = self._post_json(self._url('/exec/{0}/start'.format(exec_id)), data=data, stream=stream) - self._raise_for_status(res) - if stream: - return self._multiplexed_response_stream_helper(res) - elif six.PY3: - return bytes().join( - [x for x in self._multiplexed_buffer_helper(res)] - ) - else: - return str().join( - [x for x in self._multiplexed_buffer_helper(res)] - ) + return self._get_result_tty(stream, res, tty) @check_resource def export(self, container): @@ -588,16 +557,7 @@ class Client(clientbase.ClientBase): params['tail'] = tail url = self._url("/containers/{0}/logs".format(container)) res = self._get(url, params=params, stream=stream) - if stream: - return self._multiplexed_response_stream_helper(res) - elif six.PY3: - return bytes().join( - [x for x in self._multiplexed_buffer_helper(res)] - ) - else: - return str().join( - [x for x in self._multiplexed_buffer_helper(res)] - ) + return self._get_result(container, stream, res) return self.attach( container, stdout=stdout, diff --git a/docker/clientbase.py b/docker/clientbase.py index e51bf3ec..c1ae8137 100644 --- a/docker/clientbase.py +++ b/docker/clientbase.py @@ -221,6 +221,46 @@ class ClientBase(requests.Session): break yield data + def _stream_raw_result_old(self, response): + ''' Stream raw output for API versions below 1.6 ''' + self._raise_for_status(response) + for line in response.iter_lines(chunk_size=1, + decode_unicode=True): + # filter out keep-alive new lines + if line: + yield line + + def _stream_raw_result(self, response): + ''' Stream result for TTY-enabled container above API 1.6 ''' + self._raise_for_status(response) + for out in response.iter_content(chunk_size=1, decode_unicode=True): + yield out + + def _get_result(self, container, stream, res): + cont = self.inspect_container(container) + return self._get_result_tty(stream, res, cont['Config']['Tty']) + + def _get_result_tty(self, stream, res, is_tty): + # Stream multi-plexing was only introduced in API v1.6. Anything + # before that needs old-style streaming. + if utils.compare_version('1.6', self._version) < 0: + return self._stream_raw_result_old(res) + + # We should also use raw streaming (without keep-alives) + # if we're dealing with a tty-enabled container. + if is_tty: + return self._stream_raw_result(res) if stream else \ + self._result(res, binary=True) + + self._raise_for_status(res) + sep = six.binary_type() + if stream: + return self._multiplexed_response_stream_helper(res) + else: + return sep.join( + [x for x in self._multiplexed_buffer_helper(res)] + ) + def get_adapter(self, url): try: return super(ClientBase, self).get_adapter(url) diff --git a/tests/fake_api.py b/tests/fake_api.py index d201838e..199b4f64 100644 --- a/tests/fake_api.py +++ b/tests/fake_api.py @@ -129,11 +129,11 @@ def post_fake_create_container(): return status_code, response -def get_fake_inspect_container(): +def get_fake_inspect_container(tty=False): status_code = 200 response = { 'Id': FAKE_CONTAINER_ID, - 'Config': {'Privileged': True}, + 'Config': {'Privileged': True, 'Tty': tty}, 'ID': FAKE_CONTAINER_ID, 'Image': 'busybox:latest', "State": { diff --git a/tests/test.py b/tests/test.py index 2e3b652b..f6535b2e 100644 --- a/tests/test.py +++ b/tests/test.py @@ -69,6 +69,14 @@ def fake_resolve_authconfig(authconfig, registry=None): return None +def fake_inspect_container(self, container, tty=False): + return fake_api.get_fake_inspect_container(tty=tty)[1] + + +def fake_inspect_container_tty(self, container): + return fake_inspect_container(self, container, tty=True) + + def fake_resp(url, data=None, **kwargs): status_code, content = fake_api.fake_responses[url]() return response(status_code=status_code, content=content) @@ -1546,7 +1554,9 @@ class DockerClientTest(Cleanup, base.BaseTestCase): def test_logs(self): try: - logs = self.client.logs(fake_api.FAKE_CONTAINER_ID) + with mock.patch('docker.Client.inspect_container', + fake_inspect_container): + logs = self.client.logs(fake_api.FAKE_CONTAINER_ID) except Exception as e: self.fail('Command should not raise exception: {0}'.format(e)) @@ -1565,7 +1575,9 @@ class DockerClientTest(Cleanup, base.BaseTestCase): def test_logs_with_dict_instead_of_id(self): try: - logs = self.client.logs({'Id': fake_api.FAKE_CONTAINER_ID}) + with mock.patch('docker.Client.inspect_container', + fake_inspect_container): + logs = self.client.logs({'Id': fake_api.FAKE_CONTAINER_ID}) except Exception as e: self.fail('Command should not raise exception: {0}'.format(e)) @@ -1584,7 +1596,9 @@ class DockerClientTest(Cleanup, base.BaseTestCase): def test_log_streaming(self): try: - self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=True) + with mock.patch('docker.Client.inspect_container', + fake_inspect_container): + self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=True) except Exception as e: self.fail('Command should not raise exception: {0}'.format(e)) @@ -1598,7 +1612,10 @@ class DockerClientTest(Cleanup, base.BaseTestCase): def test_log_tail(self): try: - self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=False, tail=10) + with mock.patch('docker.Client.inspect_container', + fake_inspect_container): + self.client.logs(fake_api.FAKE_CONTAINER_ID, stream=False, + tail=10) except Exception as e: self.fail('Command should not raise exception: {0}'.format(e)) @@ -1610,6 +1627,27 @@ class DockerClientTest(Cleanup, base.BaseTestCase): stream=False ) + def test_log_tty(self): + try: + m = mock.Mock() + with mock.patch('docker.Client.inspect_container', + fake_inspect_container_tty): + with mock.patch('docker.Client._stream_raw_result', + m): + self.client.logs(fake_api.FAKE_CONTAINER_ID, + stream=True) + except Exception as e: + self.fail('Command should not raise exception: {0}'.format(e)) + + self.assertTrue(m.called) + fake_request.assert_called_with( + url_prefix + 'containers/3cc2351ab11b/logs', + params={'timestamps': 0, 'follow': 1, 'stderr': 1, 'stdout': 1, + 'tail': 'all'}, + timeout=DEFAULT_TIMEOUT_SECONDS, + stream=True + ) + def test_diff(self): try: self.client.diff(fake_api.FAKE_CONTAINER_ID) From 69ef9e5668254adf8c951571660b11d4c89b2529 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Jul 2015 13:38:41 -0700 Subject: [PATCH 10/43] Added missing `Client.load_image` docs Clarified `Client.wait` docs --- docs/api.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/api.md b/docs/api.md index 5a3b3222..f1e2d034 100644 --- a/docs/api.md +++ b/docs/api.md @@ -513,6 +513,15 @@ Kill a container or send a signal to a container * container (str): The container to kill * signal (str or int): The singal to send. Defaults to `SIGKILL` +## load_image + +Load an image that was previously saved using `Client.get_image` +(or `docker save`). Similar to `docker load`. + +**Params**: + +* data (binary): Image data to be loaded + ## login Nearly identical to the `docker login` command, but non-interactive. @@ -828,10 +837,13 @@ Nearly identical to the `docker version` command. ## wait Identical to the `docker wait` command. Block until a container stops, then -print its exit code. Returns the value `-1` if no `StatusCode` is returned by -the API. +return its exit code. Returns the value `-1` if the API responds without a +`StatusCode` attribute. -If `container` a dict, the `Id` key is used. +If `container` is a dict, the `Id` key is used. + +If the timeout value is exceeded, a `requests.exceptions.ReadTimeout` +exception will be raised. **Params**: From 826c9e676af8052e924ec3c280ba9c9f73b1cda5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Jul 2015 15:08:13 -0700 Subject: [PATCH 11/43] Removed execute() method --- docker/client.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/docker/client.py b/docker/client.py index 17b7da10..e52cb538 100644 --- a/docker/client.py +++ b/docker/client.py @@ -271,19 +271,6 @@ class Client(clientbase.ClientBase): params=params, stream=True), decode=decode) - @check_resource - def execute(self, container, cmd, detach=False, stdout=True, stderr=True, - stream=False, tty=False): - warnings.warn( - 'Client.execute is being deprecated. Please use exec_create & ' - 'exec_start instead', DeprecationWarning - ) - create_res = self.exec_create( - container, cmd, stdout, stderr, tty - ) - - return self.exec_start(create_res, detach, tty, stream) - def exec_create(self, container, cmd, stdout=True, stderr=True, tty=False, privileged=False): if utils.compare_version('1.15', self._version) < 0: From 7d85f684608b46c7a7822d6c0ca8577258e82623 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Jul 2015 15:08:35 -0700 Subject: [PATCH 12/43] Bumped version to 1.3.0 --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 88859a6c..42ddd98e 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.3.0-dev" +version = "1.3.0" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From aa20f40fcddee4ea1b58a4d0150dfee359f296f0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Jul 2015 15:08:43 -0700 Subject: [PATCH 13/43] Updated changelog --- docs/change_log.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/change_log.md b/docs/change_log.md index aac4acb1..305721cc 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -1,6 +1,55 @@ Change Log ========== +1.3.0 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/issues?q=milestone%3A1.3.0+is%3Aclosed) + +### Deprecation warning + +* As announced in the 1.2.0 release, `Client.execute` has been removed in favor of + `Client.exec_create` and `Client.exec_start`. + +### Features + +* `extra_hosts` parameter in host config can now also be provided as a list. +* Added support for `memory_limit` and `memswap_limit` in host config to + comply with recent deprecations. +* Added support for `volume_driver` in `Client.create_container` +* Added support for advanced modes in volume binds (using the `mode` key) +* Added support for `decode` in `Client.build` (decodes JSON stream on the fly) +* docker-py will now look for login configuration under the new config path, + and fall back to the old `~/.dockercfg` path if not present. + +### Bugfixes + +* Configuration file lookup now also work on platforms that don't define a + `$HOME` environment variable. +* Fixed an issue where pinging a v2 private registry wasn't working properly, + preventing users from pushing and pulling. +* `pull` parameter in `Client.build` now defaults to `False`. Fixes a bug where + the default options would try to force a pull of non-remote images. +* Fixed a bug where getting logs from tty-enabled containers wasn't working + properly with more recent versions of Docker +* `Client.push` and `Client.pull` will now raise exceptions if the HTTP + status indicates an error. +* Fixed a bug with adapter lookup when using the Unix socket adapter + (this affected some weird edge cases, see issue #647 for details) +* Fixed a bug where providing `timeout=None` to `Client.stop` would result + in an exception despite the usecase being valid. +* Added `git@` to the list of valid prefixes for remote build paths. + +### Dependencies + +* The websocket-client dependency has been updated to a more recent version. + This new version also supports Python 3.x, making `attach_socket` available + on those versions as well. + +### Documentation + +* Various fixes + 1.2.3 ----- From b60d57b67b4c66d8401abe0011ff707ad3fbe8af Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Jul 2015 15:18:04 -0700 Subject: [PATCH 14/43] websocket-client for all versions --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 485d33c7..e5ed814a 100644 --- a/setup.py +++ b/setup.py @@ -9,11 +9,9 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ 'requests >= 2.5.2', 'six >= 1.3.0', + 'websocket-client >= 0.32.0', ] -if sys.version_info[0] < 3: - requirements.append('websocket-client >= 0.32.0') - exec(open('docker/version.py').read()) with open('./test-requirements.txt') as test_reqs_txt: From c846d7e1b5d70c13b9af44138b1bdcbcb9791d8a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 9 Jul 2015 11:09:48 -0700 Subject: [PATCH 15/43] Back to dev version --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index 42ddd98e..d0aad76a 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.3.0" +version = "1.4.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From f321eef0319d74d5917ec5de0b3e3b34b5d2faf8 Mon Sep 17 00:00:00 2001 From: Phil Estes Date: Tue, 14 Jul 2015 13:00:35 -0400 Subject: [PATCH 16/43] Update all client instances in integration tests to use DOCKER_HOST In testing Docker master with the latest docker-py release, a few new tests exist which don't set the base_url, which is not the default unix socket location when Docker's integration tests are running. Also, there is no guarantee syslog can be reached from a test container so I wrapped that test with the NOT_ON_HOST skip logic. Docker-DCO-1.1-Signed-off-by: Phil Estes (github: estesp) --- tests/integration_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration_test.py b/tests/integration_test.py index ac4a8717..a2718a2e 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -242,6 +242,7 @@ class TestCreateContainerWithRoBinds(BaseTestCase): self.assertFalse(inspect_data['VolumesRW'][mount_dest]) +@unittest.skipIf(NOT_ON_HOST, 'Tests running inside a container; no syslog') class TestCreateContainerWithLogConfig(BaseTestCase): def runTest(self): config = docker.utils.LogConfig( @@ -1386,7 +1387,7 @@ class TestLoadJSONConfig(BaseTestCase): class TestAutoDetectVersion(unittest.TestCase): def test_client_init(self): - client = docker.Client(version='auto') + client = docker.Client(base_url=DEFAULT_BASE_URL, version='auto') client_version = client._version api_version = client.version(api_version=False)['ApiVersion'] self.assertEqual(client_version, api_version) @@ -1395,7 +1396,7 @@ class TestAutoDetectVersion(unittest.TestCase): client.close() def test_auto_client(self): - client = docker.AutoVersionClient() + client = docker.AutoVersionClient(base_url=DEFAULT_BASE_URL) client_version = client._version api_version = client.version(api_version=False)['ApiVersion'] self.assertEqual(client_version, api_version) @@ -1403,7 +1404,7 @@ class TestAutoDetectVersion(unittest.TestCase): self.assertEqual(client_version, api_version_2) client.close() with self.assertRaises(docker.errors.DockerException): - docker.AutoVersionClient(version='1.11') + docker.AutoVersionClient(base_url=DEFAULT_BASE_URL, version='1.11') class TestConnectionTimeout(unittest.TestCase): From 2e546f7e26fec006dd18119d4857580ec9ccd540 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Thu, 16 Jul 2015 15:10:07 -0400 Subject: [PATCH 17/43] return NotFound on 404 errors This changes raises docker.errors.NotFound on 404 errors. This gives client code the ability to differentiate between "an image does not exist" and "you are using the api incorrectly". This inherits from docker.errors.APIError so it will not affect any existing code. --- docker/clientbase.py | 2 ++ docker/errors.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/docker/clientbase.py b/docker/clientbase.py index c81ee0c5..ce52ffa7 100644 --- a/docker/clientbase.py +++ b/docker/clientbase.py @@ -99,6 +99,8 @@ class ClientBase(requests.Session): try: response.raise_for_status() except requests.exceptions.HTTPError as e: + if e.response.status_code == 404: + raise errors.NotFound(e, response, explanation=explanation) raise errors.APIError(e, response, explanation=explanation) def _result(self, response, json=False, binary=False): diff --git a/docker/errors.py b/docker/errors.py index d15e3327..066406ae 100644 --- a/docker/errors.py +++ b/docker/errors.py @@ -53,6 +53,10 @@ class DockerException(Exception): pass +class NotFound(APIError): + pass + + class InvalidVersion(DockerException): pass From ebcfab0a4c1430846de1b26a114570e4eb0d6a1a Mon Sep 17 00:00:00 2001 From: Matt Outten Date: Fri, 17 Jul 2015 15:48:52 -0400 Subject: [PATCH 18/43] Switch to send full AuthConfig object for build action In order to support the docker API for version 1.7+, this command changes the way the `X-Registry-Config` header is sent when attempting to build an image. --- docker/auth/auth.py | 6 ------ docker/client.py | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/docker/auth/auth.py b/docker/auth/auth.py index 1c296155..d7fbd6b8 100644 --- a/docker/auth/auth.py +++ b/docker/auth/auth.py @@ -102,12 +102,6 @@ def encode_header(auth): return base64.b64encode(auth_json) -def encode_full_header(auth): - """ Returns the given auth block encoded for the X-Registry-Config header. - """ - return encode_header({'configs': auth}) - - def parse_auth(entries): """ Parses authentication entries diff --git a/docker/client.py b/docker/client.py index e52cb538..ce4a6571 100644 --- a/docker/client.py +++ b/docker/client.py @@ -139,7 +139,7 @@ class Client(clientbase.ClientBase): if self._auth_configs: if headers is None: headers = {} - headers['X-Registry-Config'] = auth.encode_full_header( + headers['X-Registry-Config'] = auth.encode_header( self._auth_configs ) From bec21cfce1f8694e7d36f403c99e3354902c53cb Mon Sep 17 00:00:00 2001 From: Peter Ericson Date: Sun, 19 Jul 2015 01:51:58 -0400 Subject: [PATCH 19/43] docs/api.md: singal -> signal --- docs/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index f1e2d034..f73ad84b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -511,7 +511,7 @@ Kill a container or send a signal to a container **Params**: * container (str): The container to kill -* signal (str or int): The singal to send. Defaults to `SIGKILL` +* signal (str or int): The signal to send. Defaults to `SIGKILL` ## load_image From 874be5b3338e80454c302b33eeb57cef7ab98e84 Mon Sep 17 00:00:00 2001 From: Peter Ericson Date: Sun, 19 Jul 2015 02:02:45 -0400 Subject: [PATCH 20/43] docs/api.md: rework stats params --- docs/api.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index f73ad84b..2dd175a4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -726,6 +726,10 @@ Identical to the `docker search` command. Similar to the `docker start` command, but doesn't support attach options. Use `.logs()` to recover `stdout`/`stderr`. +**Params**: + +* container (str): The container to start + **Deprecation warning:** For API version > 1.15, it is highly recommended to provide host config options in the [`host_config` parameter of `create_container`](#create_container) @@ -748,7 +752,7 @@ This will stream statistics for a specific container. **Params**: -* container (str): The container to start +* container (str): The container to stream statistics for * decode (bool): If set to true, stream will be decoded into dicts on the fly. False by default. From 33e1a58b6034f4b586a55b5b42e644c11c281d25 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 20 Jul 2015 09:36:35 +0100 Subject: [PATCH 21/43] Stop pinging registries from the client The daemon already pings the registry, so doing it on our end is redundant and error-prone. The `insecure_registry` argument to `push()`, `pull()` and `login()` has been deprecated - in the latter case, it wasn't being used anyway. The `insecure` argument to `docker.auth.resolve_repository_name()` has also been deprecated. `docker.utils.ping_registry()` has been deprecated. `docker.auth.expand_registry_url()` has been removed. Signed-off-by: Aanand Prasad --- docker/auth/__init__.py | 1 + docker/auth/auth.py | 31 ++++++-------- docker/client.py | 27 +++++++++--- docker/constants.py | 4 ++ docker/utils/utils.py | 6 +++ tests/test.py | 6 +-- tests/utils_test.py | 91 ++++++++++++++++++++++++++++++++++++++++- 7 files changed, 138 insertions(+), 28 deletions(-) diff --git a/docker/auth/__init__.py b/docker/auth/__init__.py index d068b7fa..6fc83f83 100644 --- a/docker/auth/__init__.py +++ b/docker/auth/__init__.py @@ -1,4 +1,5 @@ from .auth import ( + INDEX_NAME, INDEX_URL, encode_header, load_config, diff --git a/docker/auth/auth.py b/docker/auth/auth.py index 1c296155..7c4876a0 100644 --- a/docker/auth/auth.py +++ b/docker/auth/auth.py @@ -16,38 +16,33 @@ import base64 import fileinput import json import os +import warnings import six -from ..utils import utils from .. import errors -INDEX_URL = 'https://index.docker.io/v1/' +INDEX_NAME = 'index.docker.io' +INDEX_URL = 'https://{0}/v1/'.format(INDEX_NAME) DOCKER_CONFIG_FILENAME = os.path.join('.docker', 'config.json') LEGACY_DOCKER_CONFIG_FILENAME = '.dockercfg' -def expand_registry_url(hostname, insecure=False): - if hostname.startswith('http:') or hostname.startswith('https:'): - return hostname - if utils.ping_registry('https://' + hostname): - return 'https://' + hostname - elif insecure: - return 'http://' + hostname - else: - raise errors.DockerException( - "HTTPS endpoint unresponsive and insecure mode isn't enabled." +def resolve_repository_name(repo_name, insecure=False): + if insecure: + warnings.warn( + 'The `insecure` argument to resolve_repository_name() ' + 'is deprecated and non-functional. Please remove it.', + DeprecationWarning ) - -def resolve_repository_name(repo_name, insecure=False): if '://' in repo_name: raise errors.InvalidRepository( 'Repository name cannot contain a scheme ({0})'.format(repo_name)) parts = repo_name.split('/', 1) if '.' not in parts[0] and ':' not in parts[0] and parts[0] != 'localhost': # This is a docker index repo (ex: foo/bar or ubuntu) - return INDEX_URL, repo_name + return INDEX_NAME, repo_name if len(parts) < 2: raise errors.InvalidRepository( 'Invalid repository name ({0})'.format(repo_name)) @@ -57,7 +52,7 @@ def resolve_repository_name(repo_name, insecure=False): 'Invalid repository name, try "{0}" instead'.format(parts[1]) ) - return expand_registry_url(parts[0], insecure), parts[1] + return parts[0], parts[1] def resolve_authconfig(authconfig, registry=None): @@ -68,7 +63,7 @@ def resolve_authconfig(authconfig, registry=None): Returns None if no match was found. """ # Default to the public index server - registry = convert_to_hostname(registry) if registry else INDEX_URL + registry = convert_to_hostname(registry) if registry else INDEX_NAME if registry in authconfig: return authconfig[registry] @@ -185,7 +180,7 @@ def load_config(config_path=None): 'Invalid or empty configuration file!') username, password = decode_auth(data[0]) - conf[INDEX_URL] = { + conf[INDEX_NAME] = { 'username': username, 'password': password, 'email': data[1], diff --git a/docker/client.py b/docker/client.py index e52cb538..285718b1 100644 --- a/docker/client.py +++ b/docker/client.py @@ -25,6 +25,7 @@ from . import constants from . import errors from .auth import auth from .utils import utils, check_resource +from .constants import INSECURE_REGISTRY_DEPRECATION_WARNING class Client(clientbase.ClientBase): @@ -499,6 +500,12 @@ class Client(clientbase.ClientBase): def login(self, username, password=None, email=None, registry=None, reauth=False, insecure_registry=False, dockercfg_path=None): + if insecure_registry: + warnings.warn( + INSECURE_REGISTRY_DEPRECATION_WARNING.format('login()'), + DeprecationWarning + ) + # If we don't have any auth data so far, try reloading the config file # one more time in case anything showed up in there. # If dockercfg_path is passed check to see if the config file exists, @@ -584,11 +591,15 @@ class Client(clientbase.ClientBase): def pull(self, repository, tag=None, stream=False, insecure_registry=False, auth_config=None): + if insecure_registry: + warnings.warn( + INSECURE_REGISTRY_DEPRECATION_WARNING.format('pull()'), + DeprecationWarning + ) + if not tag: repository, tag = utils.parse_repository_tag(repository) - registry, repo_name = auth.resolve_repository_name( - repository, insecure=insecure_registry - ) + registry, repo_name = auth.resolve_repository_name(repository) if repo_name.count(":") == 1: repository, tag = repository.rsplit(":", 1) @@ -631,11 +642,15 @@ class Client(clientbase.ClientBase): def push(self, repository, tag=None, stream=False, insecure_registry=False): + if insecure_registry: + warnings.warn( + INSECURE_REGISTRY_DEPRECATION_WARNING.format('push()'), + DeprecationWarning + ) + if not tag: repository, tag = utils.parse_repository_tag(repository) - registry, repo_name = auth.resolve_repository_name( - repository, insecure=insecure_registry - ) + registry, repo_name = auth.resolve_repository_name(repository) u = self._url("/images/{0}/push".format(repository)) params = { 'tag': tag diff --git a/docker/constants.py b/docker/constants.py index f99f1922..10a2feec 100644 --- a/docker/constants.py +++ b/docker/constants.py @@ -4,3 +4,7 @@ STREAM_HEADER_SIZE_BYTES = 8 CONTAINER_LIMITS_KEYS = [ 'memory', 'memswap', 'cpushares', 'cpusetcpus' ] + +INSECURE_REGISTRY_DEPRECATION_WARNING = \ + 'The `insecure_registry` argument to {} ' \ + 'is deprecated and non-functional. Please remove it.' diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 175a7e0f..10c08de2 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -19,6 +19,7 @@ import json import shlex import tarfile import tempfile +import warnings from distutils.version import StrictVersion from fnmatch import fnmatch from datetime import datetime @@ -120,6 +121,11 @@ def compare_version(v1, v2): def ping_registry(url): + warnings.warn( + 'The `ping_registry` method is deprecated and will be removed.', + DeprecationWarning + ) + return ping(url + '/v2/', [401]) or ping(url + '/v1/_ping') diff --git a/tests/test.py b/tests/test.py index f6535b2e..3171bf98 100644 --- a/tests/test.py +++ b/tests/test.py @@ -2424,9 +2424,9 @@ class DockerClientTest(Cleanup, base.BaseTestCase): f.write('auth = {0}\n'.format(auth_)) f.write('email = sakuya@scarlet.net') cfg = docker.auth.load_config(dockercfg_path) - self.assertTrue(docker.auth.INDEX_URL in cfg) - self.assertNotEqual(cfg[docker.auth.INDEX_URL], None) - cfg = cfg[docker.auth.INDEX_URL] + self.assertTrue(docker.auth.INDEX_NAME in cfg) + self.assertNotEqual(cfg[docker.auth.INDEX_NAME], None) + cfg = cfg[docker.auth.INDEX_NAME] self.assertEqual(cfg['username'], 'sakuya') self.assertEqual(cfg['password'], 'izayoi') self.assertEqual(cfg['email'], 'sakuya@scarlet.net') diff --git a/tests/utils_test.py b/tests/utils_test.py index 716cde55..1c8729ca 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -9,7 +9,7 @@ from docker.utils import ( create_host_config, Ulimit, LogConfig, parse_bytes ) from docker.utils.ports import build_port_bindings, split_port -from docker.auth import resolve_authconfig +from docker.auth import resolve_repository_name, resolve_authconfig import base @@ -167,6 +167,61 @@ class UtilsTest(base.BaseTestCase): type=LogConfig.types.JSON, config='helloworld' )) + def test_resolve_repository_name(self): + # docker hub library image + self.assertEqual( + resolve_repository_name('image'), + ('index.docker.io', 'image'), + ) + + # docker hub image + self.assertEqual( + resolve_repository_name('username/image'), + ('index.docker.io', 'username/image'), + ) + + # private registry + self.assertEqual( + resolve_repository_name('my.registry.net/image'), + ('my.registry.net', 'image'), + ) + + # private registry with port + self.assertEqual( + resolve_repository_name('my.registry.net:5000/image'), + ('my.registry.net:5000', 'image'), + ) + + # private registry with username + self.assertEqual( + resolve_repository_name('my.registry.net/username/image'), + ('my.registry.net', 'username/image'), + ) + + # no dots but port + self.assertEqual( + resolve_repository_name('hostname:5000/image'), + ('hostname:5000', 'image'), + ) + + # no dots but port and username + self.assertEqual( + resolve_repository_name('hostname:5000/username/image'), + ('hostname:5000', 'username/image'), + ) + + # localhost + self.assertEqual( + resolve_repository_name('localhost/image'), + ('localhost', 'image'), + ) + + # localhost with username + self.assertEqual( + resolve_repository_name('localhost/username/image'), + ('localhost', 'username/image'), + ) + def test_resolve_authconfig(self): auth_config = { 'https://index.docker.io/v1/': {'auth': 'indexuser'}, @@ -231,6 +286,40 @@ class UtilsTest(base.BaseTestCase): resolve_authconfig(auth_config, 'does.not.exist') is None ) + def test_resolve_registry_and_auth(self): + auth_config = { + 'https://index.docker.io/v1/': {'auth': 'indexuser'}, + 'my.registry.net': {'auth': 'privateuser'}, + } + + # library image + image = 'image' + self.assertEqual( + resolve_authconfig(auth_config, resolve_repository_name(image)[0]), + {'auth': 'indexuser'}, + ) + + # docker hub image + image = 'username/image' + self.assertEqual( + resolve_authconfig(auth_config, resolve_repository_name(image)[0]), + {'auth': 'indexuser'}, + ) + + # private registry + image = 'my.registry.net/image' + self.assertEqual( + resolve_authconfig(auth_config, resolve_repository_name(image)[0]), + {'auth': 'privateuser'}, + ) + + # unauthenticated registry + image = 'other.registry.net/image' + self.assertEqual( + resolve_authconfig(auth_config, resolve_repository_name(image)[0]), + None, + ) + def test_split_port_with_host_ip(self): internal_port, external_port = split_port("127.0.0.1:1000:2000") self.assertEqual(internal_port, ["2000"]) From 915be6ad67fbb701273637876e51d76149bd8d8b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 20 Jul 2015 13:50:49 -0700 Subject: [PATCH 22/43] Use constant for deprecation message in auth.py --- docker/auth/auth.py | 7 ++++--- docker/client.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docker/auth/auth.py b/docker/auth/auth.py index 7c4876a0..56c42eca 100644 --- a/docker/auth/auth.py +++ b/docker/auth/auth.py @@ -20,6 +20,7 @@ import warnings import six +from .. import constants from .. import errors INDEX_NAME = 'index.docker.io' @@ -31,9 +32,9 @@ LEGACY_DOCKER_CONFIG_FILENAME = '.dockercfg' def resolve_repository_name(repo_name, insecure=False): if insecure: warnings.warn( - 'The `insecure` argument to resolve_repository_name() ' - 'is deprecated and non-functional. Please remove it.', - DeprecationWarning + constants.INSECURE_REGISTRY_DEPRECATION_WARNING.format( + 'resolve_repository_name()' + ), DeprecationWarning ) if '://' in repo_name: diff --git a/docker/client.py b/docker/client.py index 285718b1..90846895 100644 --- a/docker/client.py +++ b/docker/client.py @@ -268,9 +268,10 @@ class Client(clientbase.ClientBase): 'filters': filters } - return self._stream_helper(self.get(self._url('/events'), - params=params, stream=True), - decode=decode) + return self._stream_helper( + self.get(self._url('/events'), params=params, stream=True), + decode=decode + ) def exec_create(self, container, cmd, stdout=True, stderr=True, tty=False, privileged=False): From 4be228653df9028bef4a288148d392d7e9b0448a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 20 Jul 2015 14:06:33 -0700 Subject: [PATCH 23/43] Make build auth work with API versions < 1.19 too --- docker/client.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docker/client.py b/docker/client.py index 7fe72db2..41dd03ae 100644 --- a/docker/client.py +++ b/docker/client.py @@ -140,9 +140,14 @@ class Client(clientbase.ClientBase): if self._auth_configs: if headers is None: headers = {} - headers['X-Registry-Config'] = auth.encode_header( - self._auth_configs - ) + if utils.compare_version('1.19', self._version) >= 0: + headers['X-Registry-Config'] = auth.encode_header( + self._auth_configs + ) + else: + headers['X-Registry-Config'] = auth.encode_header({ + 'configs': self._auth_configs + }) response = self._post( u, From b712e97d4a8f74cddc7d8d4c43d630bf01d1c609 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 20 Jul 2015 14:17:06 -0700 Subject: [PATCH 24/43] Added check_resource decorator to exec_create --- docker/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/client.py b/docker/client.py index 90846895..af4b635b 100644 --- a/docker/client.py +++ b/docker/client.py @@ -273,6 +273,7 @@ class Client(clientbase.ClientBase): decode=decode ) + @check_resource def exec_create(self, container, cmd, stdout=True, stderr=True, tty=False, privileged=False): if utils.compare_version('1.15', self._version) < 0: From 657420a4d3b1611cedf37f801e597456133807a2 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 20 Jul 2015 10:34:05 +0100 Subject: [PATCH 25/43] Enforce UTC datetimes in arguments to `events()` Signed-off-by: Aanand Prasad --- docker/utils/utils.py | 6 +++--- docs/api.md | 4 ++-- tests/test.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 175a7e0f..075965c7 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -333,9 +333,9 @@ def convert_filters(filters): return json.dumps(result) -def datetime_to_timestamp(dt=datetime.now()): - """Convert a datetime in local timezone to a unix timestamp""" - delta = dt - datetime.fromtimestamp(0) +def datetime_to_timestamp(dt): + """Convert a UTC datetime to a Unix timestamp""" + delta = dt - datetime.utcfromtimestamp(0) return delta.seconds + delta.days * 24 * 3600 diff --git a/docs/api.md b/docs/api.md index f1e2d034..013ea169 100644 --- a/docs/api.md +++ b/docs/api.md @@ -251,8 +251,8 @@ function return a blocking generator you can iterate over to retrieve events as **Params**: -* since (datetime or int): get events from this point -* until (datetime or int): get events until this point +* since (UTC datetime or int): get events from this point +* until (UTC datetime or int): get events until this point * filters (dict): filter the events by event time, container or image * decode (bool): If set to true, stream will be decoded into dicts on the fly. False by default. diff --git a/tests/test.py b/tests/test.py index f6535b2e..9d4b4034 100644 --- a/tests/test.py +++ b/tests/test.py @@ -221,7 +221,7 @@ class DockerClientTest(Cleanup, base.BaseTestCase): def test_events_with_since_until(self): ts = 1356048000 - now = datetime.datetime.fromtimestamp(ts) + now = datetime.datetime.utcfromtimestamp(ts) since = now - datetime.timedelta(seconds=10) until = now + datetime.timedelta(seconds=10) try: From 7de1605e9aa0ea5704dd28248f6e9262ea1e234b Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Tue, 21 Jul 2015 12:07:22 +0100 Subject: [PATCH 26/43] Fix integration tests. Forgot to update them in 33e1a58 Signed-off-by: Paul Bellamy --- tests/integration_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_test.py b/tests/integration_test.py index a2718a2e..226ea34c 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -1357,8 +1357,8 @@ class TestLoadConfig(BaseTestCase): f.write('email = sakuya@scarlet.net') f.close() cfg = docker.auth.load_config(cfg_path) - self.assertNotEqual(cfg[docker.auth.INDEX_URL], None) - cfg = cfg[docker.auth.INDEX_URL] + self.assertNotEqual(cfg[docker.auth.INDEX_NAME], None) + cfg = cfg[docker.auth.INDEX_NAME] self.assertEqual(cfg['username'], 'sakuya') self.assertEqual(cfg['password'], 'izayoi') self.assertEqual(cfg['email'], 'sakuya@scarlet.net') From 89c048dc1702c4d4cbb6f478951386cf3809b0d3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Jul 2015 11:23:39 -0700 Subject: [PATCH 27/43] Bumped version --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index d0aad76a..bc778b93 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.4.0-dev" +version = "1.3.1" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 57b86fefe5fd23d5e365b40573528883698514e3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Jul 2015 11:23:51 -0700 Subject: [PATCH 28/43] Updated changelog for 1.3.1 --- docs/change_log.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/change_log.md b/docs/change_log.md index 305721cc..538cca4c 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -1,6 +1,28 @@ Change Log ========== +1.3.1 +----- + +[List of PRs / issues for this release](https://github.com/docker/docker-py/issues?q=milestone%3A1.3.1+is%3Aclosed) + +### Bugfixes + +* Fixed a bug where empty chunks in streams was misinterpreted as EOF. +* `datetime` arguments passed to `Client.events` parameters `since` and + `until` are now always considered to be UTC. +* Fixed a bug with Docker 1.7.x where the wrong auth headers were being passed + in `Client.build`, failing builds that depended on private images. +* `Client.exec_create` can now retrieve the `Id` key from a dictionary for its + container param. + +### Miscellaneous + +* 404 API status now raises `docker.errors.NotFound`. This exception inherits + `APIError` which was used previously. +* Docs fixes +* Test ixes + 1.3.0 ----- From 81f65e924e5d25b44a5b3017c94b427007e9d3fe Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Jul 2015 11:34:16 -0700 Subject: [PATCH 29/43] Added py3-based dockerfile --- Dockerfile | 2 +- Dockerfile-py3 | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Dockerfile-py3 diff --git a/Dockerfile b/Dockerfile index 733f96cf..20152f5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM python:2.7 -MAINTAINER Joffrey F +MAINTAINER Joffrey F ADD . /home/docker-py WORKDIR /home/docker-py RUN pip install -r test-requirements.txt diff --git a/Dockerfile-py3 b/Dockerfile-py3 new file mode 100644 index 00000000..31b979bf --- /dev/null +++ b/Dockerfile-py3 @@ -0,0 +1,6 @@ +FROM python:3.4 +MAINTAINER Joffrey F +ADD . /home/docker-py +WORKDIR /home/docker-py +RUN pip install -r test-requirements.txt +RUN pip install . From 5f0f085fc7fe7810d03a0534c92b2b5cf58da75c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 22 Jul 2015 11:34:36 -0700 Subject: [PATCH 30/43] Updated Makefile to help with release testing --- Makefile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index cf291ea9..ce749748 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build test integration-test unit-test +.PHONY: all build test integration-test unit-test build-py3 unit-test-py3 integration-test-py3 HOST_TMPDIR=test -n "$(TMPDIR)" && echo $(TMPDIR) || echo /tmp @@ -7,10 +7,19 @@ all: test build: docker build -t docker-py . -test: unit-test integration-test +build-py3: + docker build -t docker-py3 -f Dockerfile-py3 . + +test: unit-test integration-test unit-test-py3 integration-test-py3 unit-test: build docker run docker-py python tests/test.py +unit-test-py3: build-py3 + docker run docker-py3 python tests/test.py + integration-test: build docker run -e NOT_ON_HOST=true -v `$(HOST_TMPDIR)`:/tmp -v /var/run/docker.sock:/var/run/docker.sock docker-py python tests/integration_test.py + +integration-test-py3: build-py3 + docker run -e NOT_ON_HOST=true -v `$(HOST_TMPDIR)`:/tmp -v /var/run/docker.sock:/var/run/docker.sock docker-py3 python tests/integration_test.py From 8f1a82faeb46cf4eeb3c97bf8cc6a7f0a9105e89 Mon Sep 17 00:00:00 2001 From: John Howard Date: Wed, 15 Jul 2015 12:11:54 -0700 Subject: [PATCH 31/43] Add netmode (required by docker) Signed-off-by: John Howard --- docker/utils/utils.py | 10 ++++++++++ tests/integration_test.py | 28 ++++++++++++++++++---------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index a714c97c..396c2459 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -615,6 +615,16 @@ def create_container_config( if volumes_from is not None: raise errors.InvalidVersion(message.format('volumes_from')) + # NetworkMode must be present and valid in host config from 1.20 onwards + if compare_version('1.20', version) >= 0: + if host_config is None: + host_config = {'NetworkMode': 'default'} + else: + if 'NetworkMode' not in host_config: + host_config['NetworkMode'] = 'default' + elif host_config['NetworkMode'] == '': + host_config['NetworkMode'] = 'default' + return { 'Hostname': hostname, 'Domainname': domainname, diff --git a/tests/integration_test.py b/tests/integration_test.py index 226ea34c..49604332 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -181,7 +181,8 @@ class TestCreateContainerWithBinds(BaseTestCase): container = self.client.create_container( 'busybox', ['ls', mount_dest], volumes={mount_dest: {}}, - host_config=create_host_config(binds=binds) + host_config=create_host_config( + binds=binds, network_mode='none') ) container_id = container['Id'] self.client.start(container_id) @@ -221,7 +222,8 @@ class TestCreateContainerWithRoBinds(BaseTestCase): container = self.client.create_container( 'busybox', ['ls', mount_dest], volumes={mount_dest: {}}, - host_config=create_host_config(binds=binds) + host_config=create_host_config( + binds=binds, network_mode='none') ) container_id = container['Id'] self.client.start(container_id) @@ -273,7 +275,8 @@ class TestCreateContainerReadOnlyFs(BaseTestCase): def runTest(self): ctnr = self.client.create_container( 'busybox', ['mkdir', '/shrine'], - host_config=create_host_config(read_only=True) + host_config=create_host_config( + read_only=True, network_mode='none') ) self.assertIn('Id', ctnr) self.tmp_containers.append(ctnr['Id']) @@ -347,7 +350,8 @@ class TestStartContainerWithDictInsteadOfId(BaseTestCase): class TestCreateContainerPrivileged(BaseTestCase): def runTest(self): res = self.client.create_container( - 'busybox', 'true', host_config=create_host_config(privileged=True) + 'busybox', 'true', host_config=create_host_config( + privileged=True, network_mode='none') ) self.assertIn('Id', res) self.tmp_containers.append(res['Id']) @@ -591,7 +595,8 @@ class TestPort(BaseTestCase): container = self.client.create_container( 'busybox', ['sleep', '60'], ports=list(port_bindings.keys()), - host_config=create_host_config(port_bindings=port_bindings) + host_config=create_host_config( + port_bindings=port_bindings, network_mode='bridge') ) id = container['Id'] @@ -717,7 +722,8 @@ class TestCreateContainerWithVolumesFrom(BaseTestCase): ) res2 = self.client.create_container( 'busybox', 'cat', detach=True, stdin_open=True, - host_config=create_host_config(volumes_from=vol_names) + host_config=create_host_config( + volumes_from=vol_names, network_mode='none') ) container3_id = res2['Id'] self.tmp_containers.append(container3_id) @@ -760,7 +766,8 @@ class TestCreateContainerWithLinks(BaseTestCase): res2 = self.client.create_container( 'busybox', 'env', host_config=create_host_config( - links={link_path1: link_alias1, link_path2: link_alias2} + links={link_path1: link_alias1, link_path2: link_alias2}, + network_mode='none' ) ) container3_id = res2['Id'] @@ -781,7 +788,8 @@ class TestRestartingContainer(BaseTestCase): def runTest(self): container = self.client.create_container( 'busybox', ['sleep', '2'], host_config=create_host_config( - restart_policy={"Name": "always", "MaximumRetryCount": 0} + restart_policy={"Name": "always", "MaximumRetryCount": 0}, + network_mode='none' ) ) id = container['Id'] @@ -910,7 +918,7 @@ class TestCreateContainerWithHostPidMode(BaseTestCase): def runTest(self): ctnr = self.client.create_container( 'busybox', 'true', host_config=create_host_config( - pid_mode='host' + pid_mode='host', network_mode='none' ) ) self.assertIn('Id', ctnr) @@ -945,7 +953,7 @@ class TestRemoveLink(BaseTestCase): container2 = self.client.create_container( 'busybox', 'cat', host_config=create_host_config( - links={link_path: link_alias} + links={link_path: link_alias}, network_mode='none' ) ) container2_id = container2['Id'] From 78ccfc88e873b621a5b4c5873155751e40aa9c1c Mon Sep 17 00:00:00 2001 From: Alex Chan Date: Wed, 29 Jul 2015 14:00:18 +0100 Subject: [PATCH 32/43] Fix a few typos in the docs Signed-off-by: Alex Chan --- docs/api.md | 8 ++++---- docs/change_log.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/api.md b/docs/api.md index d35e0832..bccd5268 100644 --- a/docs/api.md +++ b/docs/api.md @@ -30,7 +30,7 @@ the entire backlog. * container (str): The container to attach to * stdout (bool): Get STDOUT * stderr (bool): Get STDERR -* stream (bool): Return an interator +* stream (bool): Return an iterator * logs (bool): Get all previous output **Returns** (generator or str): The logs or output for the image @@ -70,7 +70,7 @@ correct value (e.g `gzip`). - memory (int): set memory limit for build - memswap (int): Total memory (memory + swap), -1 to disable swap - cpushares (int): CPU shares (relative weight) - - cpusetcpus (str): CPUs in which to allow exection, e.g., `"0-3"`, `"0,1"` + - cpusetcpus (str): CPUs in which to allow execution, e.g., `"0-3"`, `"0,1"` * decode (bool): If set to `True`, the returned stream will be decoded into dicts on the fly. Default `False`. @@ -123,7 +123,7 @@ Identical to the `docker commit` command. * tag (str): The tag to push * message (str): A commit message * author (str): The name of the author -* conf (dict): The configuraton for the container. See the [Docker remote api]( +* conf (dict): The configuration for the container. See the [Docker remote api]( https://docs.docker.com/reference/api/docker_remote_api/) for full details. ## containers @@ -397,7 +397,7 @@ src will be treated as a URL instead to fetch the image from. You can also pass an open file handle as 'src', in which case the data will be read from that file. -If `src` is unset but `image` is set, the `image` paramater will be taken as +If `src` is unset but `image` is set, the `image` parameter will be taken as the name of an existing image to import from. **Params**: diff --git a/docs/change_log.md b/docs/change_log.md index 538cca4c..5e91861a 100644 --- a/docs/change_log.md +++ b/docs/change_log.md @@ -21,7 +21,7 @@ Change Log * 404 API status now raises `docker.errors.NotFound`. This exception inherits `APIError` which was used previously. * Docs fixes -* Test ixes +* Test fixes 1.3.0 ----- From 80a97cf2ea39fdf0d4b14a30128aee22ef17432a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 29 Jul 2015 12:40:54 -0700 Subject: [PATCH 33/43] development version suffix --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index bc778b93..d0aad76a 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.3.1" +version = "1.4.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From daea185a2397c3b92c80a6cfe7752a9aff672141 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 29 Jul 2015 12:43:39 -0700 Subject: [PATCH 34/43] nit: parenthesis alignment --- tests/integration_test.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/integration_test.py b/tests/integration_test.py index 49604332..59919dab 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -182,7 +182,8 @@ class TestCreateContainerWithBinds(BaseTestCase): 'busybox', ['ls', mount_dest], volumes={mount_dest: {}}, host_config=create_host_config( - binds=binds, network_mode='none') + binds=binds, network_mode='none' + ) ) container_id = container['Id'] self.client.start(container_id) @@ -223,7 +224,8 @@ class TestCreateContainerWithRoBinds(BaseTestCase): 'busybox', ['ls', mount_dest], volumes={mount_dest: {}}, host_config=create_host_config( - binds=binds, network_mode='none') + binds=binds, network_mode='none' + ) ) container_id = container['Id'] self.client.start(container_id) @@ -276,7 +278,8 @@ class TestCreateContainerReadOnlyFs(BaseTestCase): ctnr = self.client.create_container( 'busybox', ['mkdir', '/shrine'], host_config=create_host_config( - read_only=True, network_mode='none') + read_only=True, network_mode='none' + ) ) self.assertIn('Id', ctnr) self.tmp_containers.append(ctnr['Id']) @@ -351,7 +354,8 @@ class TestCreateContainerPrivileged(BaseTestCase): def runTest(self): res = self.client.create_container( 'busybox', 'true', host_config=create_host_config( - privileged=True, network_mode='none') + privileged=True, network_mode='none' + ) ) self.assertIn('Id', res) self.tmp_containers.append(res['Id']) @@ -596,7 +600,8 @@ class TestPort(BaseTestCase): container = self.client.create_container( 'busybox', ['sleep', '60'], ports=list(port_bindings.keys()), host_config=create_host_config( - port_bindings=port_bindings, network_mode='bridge') + port_bindings=port_bindings, network_mode='bridge' + ) ) id = container['Id'] @@ -723,7 +728,8 @@ class TestCreateContainerWithVolumesFrom(BaseTestCase): res2 = self.client.create_container( 'busybox', 'cat', detach=True, stdin_open=True, host_config=create_host_config( - volumes_from=vol_names, network_mode='none') + volumes_from=vol_names, network_mode='none' + ) ) container3_id = res2['Id'] self.tmp_containers.append(container3_id) From e32726e83d1375e2ed64747428ba8ca294b90df8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 29 Jul 2015 12:45:15 -0700 Subject: [PATCH 35/43] Moved network_mode default to create_host_config ; small fix to preserve blank host_config in start if no option is provided --- docker/client.py | 2 +- docker/utils/utils.py | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/docker/client.py b/docker/client.py index e4712c23..4e84dd60 100644 --- a/docker/client.py +++ b/docker/client.py @@ -780,7 +780,7 @@ class Client(clientbase.ClientBase): publish_all_ports=publish_all_ports, links=links, dns=dns, privileged=privileged, dns_search=dns_search, cap_add=cap_add, cap_drop=cap_drop, volumes_from=volumes_from, devices=devices, - network_mode=network_mode, restart_policy=restart_policy, + network_mode=network_mode or '', restart_policy=restart_policy, extra_hosts=extra_hosts, read_only=read_only, pid_mode=pid_mode, ipc_mode=ipc_mode, security_opt=security_opt, ulimits=ulimits ) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 396c2459..98d17ef5 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -428,6 +428,8 @@ def create_host_config( if network_mode: host_config['NetworkMode'] = network_mode + elif network_mode is None: + host_config['NetworkMode'] = 'default' if restart_policy: host_config['RestartPolicy'] = restart_policy @@ -615,16 +617,6 @@ def create_container_config( if volumes_from is not None: raise errors.InvalidVersion(message.format('volumes_from')) - # NetworkMode must be present and valid in host config from 1.20 onwards - if compare_version('1.20', version) >= 0: - if host_config is None: - host_config = {'NetworkMode': 'default'} - else: - if 'NetworkMode' not in host_config: - host_config['NetworkMode'] = 'default' - elif host_config['NetworkMode'] == '': - host_config['NetworkMode'] = 'default' - return { 'Hostname': hostname, 'Domainname': domainname, From 2b7c31e0f7bc4f868e7ea9cdefd4b7eed6ee7a1a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 29 Jul 2015 14:23:10 -0700 Subject: [PATCH 36/43] Fixed empty_host_config test --- tests/utils_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils_test.py b/tests/utils_test.py index 1c8729ca..fd9d7f3f 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -107,7 +107,7 @@ class UtilsTest(base.BaseTestCase): self.assertEqual(convert_filters(filters), expected) def test_create_empty_host_config(self): - empty_config = create_host_config() + empty_config = create_host_config(network_mode='') self.assertEqual(empty_config, {}) def test_create_host_config_dict_ulimit(self): From d400717ff895f9786a83a6d0d46aa365660c0953 Mon Sep 17 00:00:00 2001 From: Viktor Petersson Date: Thu, 30 Jul 2015 14:39:37 +0100 Subject: [PATCH 37/43] Squashed commit of the following: commit 4f053a06c1e9e3f63fd5afde60322f676acbdf45 Merge: 9177380 07a99ea Author: Viktor Petersson Date: Thu Jul 30 14:37:16 2015 +0100 Merge branch 'master' into fixes commit 9177380ae9abf4ca01cf7a2a8b7de6640ed5f5d5 Author: Viktor Petersson Date: Thu Jul 30 14:00:51 2015 +0100 Tweaks exception message. commit 6a5832e2f6655835a6b87897d7ec82d6ef5faa50 Author: Viktor Petersson Date: Thu Jul 30 13:17:32 2015 +0100 Simplifies logic as per feedback. commit f750eddc34e5356dcfb53c16a7f98a99e817fa53 Author: Viktor Petersson Date: Thu Jul 30 11:09:14 2015 +0100 Move return from list to dict. Adds exception handling. commit 8e50f57cceb6370b7f7b41624f50d5a6835301a0 Author: Viktor Petersson Date: Thu Jul 30 10:15:58 2015 +0100 Reverts change to .gitignore. commit 5ba2c1b29706ddb74bef9c24c2a6e64bb369db22 Author: Viktor Petersson Date: Wed Jul 29 21:15:21 2015 +0100 Fixes feedback. Adds three unittests. commit e1c719e61993fdaec0f22ba0de706af80587dc16 Author: Viktor Petersson Date: Wed Jul 29 17:00:16 2015 +0100 WIP Adds test for parse_env_file commit 4448ae72e55d889d9d194a2c0303d182cb157d4c Author: Viktor Petersson Date: Wed Jul 29 16:42:49 2015 +0100 Excludes coverage files. commit 19a5d01615dd9b7ee939f54f256e5bae89a94ee1 Author: Viktor Petersson Date: Wed Jul 29 16:42:42 2015 +0100 Switch fixes logic. commit a8094c63a9f4351ae71393ea5603aa11c5c5bc94 Author: Viktor Petersson Date: Wed Jul 29 11:45:56 2015 +0100 Implements logic for envfile parsing from Docker-cli Ref: https://github.com/docker/docker/blob/master/opts/envfile.go#L19-L51 commit ea9bfd95dfafe4023c58ab37f990158f73eb2e0d Author: Viktor Petersson Date: Wed Jul 29 11:41:23 2015 +0100 Replaces CSV module with manual splitting. commit a001d28ff48309d2b2338aaf27253fdfaa0f6c4b Author: Viktor Petersson Date: Wed Jul 29 11:35:37 2015 +0100 Removes isinstance on filename. commit 419d5961f6103df9166be3a9baa549276c12223d Author: Viktor Petersson Date: Tue Jul 28 22:39:33 2015 +0100 Reflects @aanand's feedback. commit e81e3c8ed797ff939843d2485bf15525e85e890d Author: Viktor Petersson Date: Tue Jul 28 15:43:32 2015 +0100 Typo fix. commit 2898389cada2bfca64bdfa71359aebeb3b5b6d1b Author: Viktor Petersson Date: Tue Jul 28 15:41:08 2015 +0100 Refs #565. Adds minimal implementation of env_file client-side support. --- docker/utils/__init__.py | 2 +- docker/utils/utils.py | 27 +++++++++++++++++++++++++++ docs/api.md | 21 +++++++++++++++++++++ tests/utils_test.py | 36 +++++++++++++++++++++++++++++++++++- 4 files changed, 84 insertions(+), 2 deletions(-) diff --git a/docker/utils/__init__.py b/docker/utils/__init__.py index 81cc8a68..6189ed83 100644 --- a/docker/utils/__init__.py +++ b/docker/utils/__init__.py @@ -2,7 +2,7 @@ from .utils import ( compare_version, convert_port_bindings, convert_volume_binds, mkbuildcontext, tar, parse_repository_tag, parse_host, kwargs_from_env, convert_filters, create_host_config, - create_container_config, parse_bytes, ping_registry + create_container_config, parse_bytes, ping_registry, parse_env_file ) # flake8: noqa from .types import Ulimit, LogConfig # flake8: noqa diff --git a/docker/utils/utils.py b/docker/utils/utils.py index a714c97c..3cd6760a 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -518,6 +518,32 @@ def create_host_config( return host_config +def parse_env_file(env_file): + """ + Reads a line-separated environment file. + The format of each line should be "key=value". + """ + environment = {} + + with open(env_file, 'r') as f: + for line in f: + + if line[0] == '#': + continue + + parse_line = line.strip().split('=') + if len(parse_line) == 2: + k = parse_line[0] + v = parse_line[1] + environment[k] = v + else: + raise errors.DockerException( + 'Invalid line in environment file {0}:\n{1}'.format( + env_file, line)) + + return environment + + def create_container_config( version, image, command, hostname=None, user=None, detach=False, stdin_open=False, tty=False, mem_limit=None, ports=None, environment=None, @@ -528,6 +554,7 @@ def create_container_config( ): if isinstance(command, six.string_types): command = shlex.split(str(command)) + if isinstance(environment, dict): environment = [ six.text_type('{0}={1}').format(k, v) diff --git a/docs/api.md b/docs/api.md index bccd5268..b9b29c5a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -234,6 +234,27 @@ from. Optionally a single string joining container id's with commas 'Warnings': None} ``` +### parse_env_file + +A utility for parsing an environment file. + +The expected format of the file is as follows: + +``` +USERNAME=jdoe +PASSWORD=secret +``` + +The utility can be used as follows: + +```python +>> import docker.utils +>> my_envs = docker.utils.parse_env_file('/path/to/file') +>> docker.utils.create_container_config('1.18', '_mongodb', 'foobar', environment=my_envs) +``` + +You can now use this with 'environment' for `create_container`. + ## diff Inspect changes on a container's filesystem diff --git a/tests/utils_test.py b/tests/utils_test.py index 1c8729ca..a73d949e 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -1,12 +1,13 @@ import os import os.path import unittest +import tempfile from docker.client import Client from docker.errors import DockerException from docker.utils import ( parse_repository_tag, parse_host, convert_filters, kwargs_from_env, - create_host_config, Ulimit, LogConfig, parse_bytes + create_host_config, Ulimit, LogConfig, parse_bytes, parse_env_file ) from docker.utils.ports import build_port_bindings, split_port from docker.auth import resolve_repository_name, resolve_authconfig @@ -17,6 +18,17 @@ import base class UtilsTest(base.BaseTestCase): longMessage = True + def generate_tempfile(self, file_content=None): + """ + Generates a temporary file for tests with the content + of 'file_content' and returns the filename. + Don't forget to unlink the file with os.unlink() after. + """ + local_tempfile = tempfile.NamedTemporaryFile(delete=False) + local_tempfile.write(file_content.encode('UTF-8')) + local_tempfile.close() + return local_tempfile.name + def setUp(self): self.os_environ = os.environ.copy() @@ -95,6 +107,28 @@ class UtilsTest(base.BaseTestCase): except TypeError as e: self.fail(e) + def test_parse_env_file_proper(self): + env_file = self.generate_tempfile( + file_content='USER=jdoe\nPASS=secret') + get_parse_env_file = parse_env_file(env_file) + self.assertEqual(get_parse_env_file, + {'USER': 'jdoe', 'PASS': 'secret'}) + os.unlink(env_file) + + def test_parse_env_file_commented_line(self): + env_file = self.generate_tempfile( + file_content='USER=jdoe\n#PASS=secret') + get_parse_env_file = parse_env_file((env_file)) + self.assertEqual(get_parse_env_file, {'USER': 'jdoe'}) + os.unlink(env_file) + + def test_parse_env_file_invalid_line(self): + env_file = self.generate_tempfile( + file_content='USER jdoe') + self.assertRaises( + DockerException, parse_env_file, env_file) + os.unlink(env_file) + def test_convert_filters(self): tests = [ ({'dangling': True}, '{"dangling": ["true"]}'), From 5803896e7c35dbdd4c842cfb7ee4a216b540f51e Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Sat, 1 Aug 2015 12:35:38 -0700 Subject: [PATCH 38/43] Add documentation for mem_limit and memswap_limit `create_host_config` has 2 additional parameters that were previously undocumented. --- docs/hostconfig.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/hostconfig.md b/docs/hostconfig.md index 001be172..c2a4edae 100644 --- a/docs/hostconfig.md +++ b/docs/hostconfig.md @@ -91,6 +91,8 @@ for example: * ulimits (list): A list of dicts or `docker.utils.Ulimit` objects. A list of ulimits to be set in the container. * log_config (`docker.utils.LogConfig` or dict): Logging configuration to container +* mem_limit (str or num): Maximum amount of memory container is allowed to consume. (e.g. `'1g'`) +* memswap_limit (str or num): Maximum amount of memory + swap a container is allowed to consume. **Returns** (dict) HostConfig dictionary From 275b45ddefbc9e6bf3f28074c70119933fa898b6 Mon Sep 17 00:00:00 2001 From: Trent Hauck Date: Sun, 2 Aug 2015 10:49:57 -0700 Subject: [PATCH 39/43] bump six req to >=1.4 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b23ea488..72c255d3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ requests==2.5.3 -six>=1.3.0 +six>=1.4.0 websocket-client==0.32.0 diff --git a/setup.py b/setup.py index e5ed814a..c2f9ae96 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ SOURCE_DIR = os.path.join(ROOT_DIR) requirements = [ 'requests >= 2.5.2', - 'six >= 1.3.0', + 'six >= 1.4.0', 'websocket-client >= 0.32.0', ] From 85bdc9c2bdb496e475836e5e16175a36945cbe10 Mon Sep 17 00:00:00 2001 From: Erik Cederberg Date: Sat, 25 Jul 2015 21:40:38 +0200 Subject: [PATCH 40/43] Add journald log driver support Signed-off-by: Erik Cederberg --- docker/utils/types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/utils/types.py b/docker/utils/types.py index d742fd0a..ca674671 100644 --- a/docker/utils/types.py +++ b/docker/utils/types.py @@ -5,9 +5,10 @@ class LogConfigTypesEnum(object): _values = ( 'json-file', 'syslog', + 'journald', 'none' ) - JSON, SYSLOG, NONE = _values + JSON, SYSLOG, JOURNALD, NONE = _values class DictType(dict): From 69027f040c60f760ab6afdf707ea1d96301b65d9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 4 Aug 2015 11:27:36 -0700 Subject: [PATCH 41/43] dev version --- docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/version.py b/docker/version.py index bc778b93..d0aad76a 100644 --- a/docker/version.py +++ b/docker/version.py @@ -1,2 +1,2 @@ -version = "1.3.1" +version = "1.4.0-dev" version_info = tuple([int(d) for d in version.split("-")[0].split(".")]) From 20a7e860b4b0ee37308f747b06c10380e584039f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 4 Aug 2015 11:31:18 -0700 Subject: [PATCH 42/43] unpack key-value pair --- docker/utils/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/utils/utils.py b/docker/utils/utils.py index 3cd6760a..315f170e 100644 --- a/docker/utils/utils.py +++ b/docker/utils/utils.py @@ -533,8 +533,7 @@ def parse_env_file(env_file): parse_line = line.strip().split('=') if len(parse_line) == 2: - k = parse_line[0] - v = parse_line[1] + k, v = parse_line environment[k] = v else: raise errors.DockerException( From 1eaf221391fbaf33d855f7821f55dcf5ec8027bc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 10 Aug 2015 10:45:26 -0700 Subject: [PATCH 43/43] Explicited start config tricks --- docker/client.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/docker/client.py b/docker/client.py index 4e84dd60..f79ec7bb 100644 --- a/docker/client.py +++ b/docker/client.py @@ -733,7 +733,7 @@ class Client(clientbase.ClientBase): @check_resource def start(self, container, binds=None, port_bindings=None, lxc_conf=None, - publish_all_ports=False, links=None, privileged=False, + publish_all_ports=None, links=None, privileged=None, dns=None, dns_search=None, volumes_from=None, network_mode=None, restart_policy=None, cap_add=None, cap_drop=None, devices=None, extra_hosts=None, read_only=None, pid_mode=None, ipc_mode=None, @@ -775,25 +775,27 @@ class Client(clientbase.ClientBase): 'ulimits is only supported for API version >= 1.18' ) - start_config = utils.create_host_config( + start_config_kwargs = dict( binds=binds, port_bindings=port_bindings, lxc_conf=lxc_conf, publish_all_ports=publish_all_ports, links=links, dns=dns, privileged=privileged, dns_search=dns_search, cap_add=cap_add, cap_drop=cap_drop, volumes_from=volumes_from, devices=devices, - network_mode=network_mode or '', restart_policy=restart_policy, + network_mode=network_mode, restart_policy=restart_policy, extra_hosts=extra_hosts, read_only=read_only, pid_mode=pid_mode, ipc_mode=ipc_mode, security_opt=security_opt, ulimits=ulimits ) + start_config = None + + if any(v is not None for v in start_config_kwargs.values()): + if utils.compare_version('1.15', self._version) > 0: + warnings.warn( + 'Passing host config parameters in start() is deprecated. ' + 'Please use host_config in create_container instead!', + DeprecationWarning + ) + start_config = utils.create_host_config(**start_config_kwargs) url = self._url("/containers/{0}/start".format(container)) - if not start_config: - start_config = None - elif utils.compare_version('1.15', self._version) > 0: - warnings.warn( - 'Passing host config parameters in start() is deprecated. ' - 'Please use host_config in create_container instead!', - DeprecationWarning - ) res = self._post_json(url, data=start_config) self._raise_for_status(res)