From 809443d6d03e1ec687c01e546ddd9031b56ce40c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 19 Aug 2014 17:36:46 -0400 Subject: [PATCH 1/8] Support python 3 Signed-off-by: Daniel Nephin --- Dockerfile | 4 +- MANIFEST.in | 2 +- compose/cli/log_printer.py | 6 ++- compose/cli/utils.py | 3 +- compose/progress_stream.py | 9 ++-- compose/project.py | 1 + compose/service.py | 2 +- ...ements-dev.txt => requirements-dev-py2.txt | 0 requirements-dev-py3.txt | 2 + setup.py | 5 +- tests/__init__.py | 5 ++ tests/integration/cli_test.py | 46 +++++++++---------- tests/integration/service_test.py | 24 +++++----- tests/unit/cli/docker_client_test.py | 3 +- tests/unit/cli/verbose_proxy_test.py | 7 ++- tests/unit/cli_test.py | 2 +- tests/unit/config_test.py | 18 ++++---- tests/unit/container_test.py | 2 +- tests/unit/log_printer_test.py | 9 ++-- tests/unit/project_test.py | 2 +- tests/unit/service_test.py | 2 +- tests/unit/split_buffer_test.py | 36 +++++++-------- tox.ini | 23 +++++++++- 23 files changed, 128 insertions(+), 85 deletions(-) rename requirements-dev.txt => requirements-dev-py2.txt (100%) create mode 100644 requirements-dev-py3.txt diff --git a/Dockerfile b/Dockerfile index a4cc99fea0..1986ac5a5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,8 +66,8 @@ WORKDIR /code/ ADD requirements.txt /code/ RUN pip install -r requirements.txt -ADD requirements-dev.txt /code/ -RUN pip install -r requirements-dev.txt +ADD requirements-dev-py2.txt /code/ +RUN pip install -r requirements-dev-py2.txt RUN pip install tox==2.1.1 diff --git a/MANIFEST.in b/MANIFEST.in index 7d48d347a8..7420485961 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,7 @@ include Dockerfile include LICENSE include requirements.txt -include requirements-dev.txt +include requirements-dev*.txt include tox.ini include *.md include compose/config/schema.json diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index ef484ca6cb..c7d0b638f8 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -4,6 +4,8 @@ from __future__ import unicode_literals import sys from itertools import cycle +import six + from . import colors from .multiplexer import Multiplexer from .utils import split_buffer @@ -20,6 +22,8 @@ class LogPrinter(object): def run(self): mux = Multiplexer(self.generators) for line in mux.loop(): + if isinstance(line, six.text_type) and not six.PY3: + line = line.encode('utf-8') self.output.write(line) def _calculate_prefix_width(self, containers): @@ -52,7 +56,7 @@ class LogPrinter(object): return generators def _make_log_generator(self, container, color_fn): - prefix = color_fn(self._generate_prefix(container)).encode('utf-8') + prefix = color_fn(self._generate_prefix(container)) # Attach to container before log printer starts running line_generator = split_buffer(self._attach(container), '\n') diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 1bb497cd80..b6c83f9e1f 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -9,6 +9,7 @@ import ssl import subprocess from docker import version as docker_py_version +from six.moves import input from .. import __version__ @@ -23,7 +24,7 @@ def yesno(prompt, default=None): Unrecognised input (anything other than "y", "n", "yes", "no" or "") will return None. """ - answer = raw_input(prompt).strip().lower() + answer = input(prompt).strip().lower() if answer == "y" or answer == "yes": return True diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 1ccdb861bd..582c09fb9e 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -1,6 +1,7 @@ import codecs import json -import os + +import six class StreamOutputError(Exception): @@ -8,8 +9,9 @@ class StreamOutputError(Exception): def stream_output(output, stream): - is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno()) - stream = codecs.getwriter('utf-8')(stream) + is_terminal = hasattr(stream, 'isatty') and stream.isatty() + if not six.PY3: + stream = codecs.getwriter('utf-8')(stream) all_events = [] lines = {} diff = 0 @@ -55,7 +57,6 @@ def print_output_event(event, stream, is_terminal): # erase current line stream.write("%c[2K\r" % 27) terminator = "\r" - pass elif 'progressDetail' in event: return diff --git a/compose/project.py b/compose/project.py index eb395297c6..d14941e72b 100644 --- a/compose/project.py +++ b/compose/project.py @@ -17,6 +17,7 @@ from .legacy import check_for_legacy_containers from .service import Service from .utils import parallel_execute + log = logging.getLogger(__name__) diff --git a/compose/service.py b/compose/service.py index 05e546c436..647516ba84 100644 --- a/compose/service.py +++ b/compose/service.py @@ -724,7 +724,7 @@ class Service(object): try: all_events = stream_output(build_output, sys.stdout) except StreamOutputError as e: - raise BuildError(self, unicode(e)) + raise BuildError(self, six.text_type(e)) # Ensure the HTTP connection is not reused for another # streaming command, as the Docker daemon can sometimes diff --git a/requirements-dev.txt b/requirements-dev-py2.txt similarity index 100% rename from requirements-dev.txt rename to requirements-dev-py2.txt diff --git a/requirements-dev-py3.txt b/requirements-dev-py3.txt new file mode 100644 index 0000000000..a2ba1c8b45 --- /dev/null +++ b/requirements-dev-py3.txt @@ -0,0 +1,2 @@ +flake8 +nose >= 1.3.0 diff --git a/setup.py b/setup.py index 2f6dad7a9b..b7fd440348 100644 --- a/setup.py +++ b/setup.py @@ -48,8 +48,11 @@ tests_require = [ ] -if sys.version_info < (2, 7): +if sys.version_info < (2, 6): tests_require.append('unittest2') +if sys.version_info[:1] < (3,): + tests_require.append('pyinstaller') + tests_require.append('mock >= 1.0.1') setup( diff --git a/tests/__init__.py b/tests/__init__.py index 08a7865e90..d3cfb86491 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,3 +4,8 @@ if sys.version_info >= (2, 7): import unittest # NOQA else: import unittest2 as unittest # NOQA + +try: + from unittest import mock +except ImportError: + import mock # NOQA diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 8bdcadd529..609370a3e1 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -5,9 +5,9 @@ import shlex import sys from operator import attrgetter -from mock import patch from six import StringIO +from .. import mock from .testcases import DockerClientTestCase from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand @@ -51,13 +51,13 @@ class CLITestCase(DockerClientTestCase): self.command.base_dir = old_base_dir # TODO: address the "Inappropriate ioctl for device" warnings in test output - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_ps(self, mock_stdout): self.project.get_service('simple').create_container() self.command.dispatch(['ps'], None) self.assertIn('simplecomposefile_simple_1', mock_stdout.getvalue()) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_ps_default_composefile(self, mock_stdout): self.command.base_dir = 'tests/fixtures/multiple-composefiles' self.command.dispatch(['up', '-d'], None) @@ -68,7 +68,7 @@ class CLITestCase(DockerClientTestCase): self.assertIn('multiplecomposefiles_another_1', output) self.assertNotIn('multiplecomposefiles_yetanother_1', output) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_ps_alternate_composefile(self, mock_stdout): config_path = os.path.abspath( 'tests/fixtures/multiple-composefiles/compose2.yml') @@ -83,19 +83,19 @@ class CLITestCase(DockerClientTestCase): self.assertNotIn('multiplecomposefiles_another_1', output) self.assertIn('multiplecomposefiles_yetanother_1', output) - @patch('compose.service.log') + @mock.patch('compose.service.log') def test_pull(self, mock_logging): self.command.dispatch(['pull'], None) mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') mock_logging.info.assert_any_call('Pulling another (busybox:latest)...') - @patch('compose.service.log') + @mock.patch('compose.service.log') def test_pull_with_digest(self, mock_logging): self.command.dispatch(['-f', 'digest.yml', 'pull'], None) mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') mock_logging.info.assert_any_call('Pulling digest (busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d)...') - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_build_no_cache(self, mock_stdout): self.command.base_dir = 'tests/fixtures/simple-dockerfile' self.command.dispatch(['build', 'simple'], None) @@ -189,7 +189,7 @@ class CLITestCase(DockerClientTestCase): self.assertFalse(config['AttachStdout']) self.assertFalse(config['AttachStdin']) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_without_links(self, mock_stdout): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['run', 'console', '/bin/true'], None) @@ -202,7 +202,7 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(config['AttachStdout']) self.assertTrue(config['AttachStdin']) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_links(self, __): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['run', 'web', '/bin/true'], None) @@ -211,14 +211,14 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(db.containers()), 1) self.assertEqual(len(console.containers()), 0) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_with_no_deps(self, __): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['run', '--no-deps', 'web', '/bin/true'], None) db = self.project.get_service('db') self.assertEqual(len(db.containers()), 0) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_does_not_recreate_linked_containers(self, __): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['up', '-d', 'db'], None) @@ -234,7 +234,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(old_ids, new_ids) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_without_command(self, _): self.command.base_dir = 'tests/fixtures/commands-composefile' self.check_build('tests/fixtures/simple-dockerfile', tag='composetest_test') @@ -255,7 +255,7 @@ class CLITestCase(DockerClientTestCase): [u'/bin/true'], ) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_entrypoint_overridden(self, _): self.command.base_dir = 'tests/fixtures/dockerfile_with_entrypoint' name = 'service' @@ -270,7 +270,7 @@ class CLITestCase(DockerClientTestCase): [u'/bin/echo', u'helloworld'], ) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_user_overridden(self, _): self.command.base_dir = 'tests/fixtures/user-composefile' name = 'service' @@ -281,7 +281,7 @@ class CLITestCase(DockerClientTestCase): container = service.containers(stopped=True, one_off=True)[0] self.assertEqual(user, container.get('Config.User')) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_user_overridden_short_form(self, _): self.command.base_dir = 'tests/fixtures/user-composefile' name = 'service' @@ -292,7 +292,7 @@ class CLITestCase(DockerClientTestCase): container = service.containers(stopped=True, one_off=True)[0] self.assertEqual(user, container.get('Config.User')) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_environement_overridden(self, _): name = 'service' self.command.base_dir = 'tests/fixtures/environment-composefile' @@ -312,7 +312,7 @@ class CLITestCase(DockerClientTestCase): # make sure a value with a = don't crash out self.assertEqual('moto=bobo', container.environment['allo']) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_without_map_ports(self, __): # create one off container self.command.base_dir = 'tests/fixtures/ports-composefile' @@ -330,7 +330,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_random, None) self.assertEqual(port_assigned, None) - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_map_ports(self, __): # create one off container @@ -353,7 +353,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_range[0], "0.0.0.0:49153") self.assertEqual(port_range[1], "0.0.0.0:49154") - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_explicitly_maped_ports(self, __): # create one off container @@ -372,7 +372,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_short, "0.0.0.0:30000") self.assertEqual(port_full, "0.0.0.0:30001") - @patch('dockerpty.start') + @mock.patch('dockerpty.start') def test_run_service_with_explicitly_maped_ip_ports(self, __): # create one off container @@ -508,7 +508,7 @@ class CLITestCase(DockerClientTestCase): self.command.dispatch(['up', '-d'], None) container = self.project.get_service('simple').get_container() - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def get_port(number, mock_stdout): self.command.dispatch(['port', 'simple', str(number)], None) return mock_stdout.getvalue().rstrip() @@ -525,7 +525,7 @@ class CLITestCase(DockerClientTestCase): self.project.containers(service_names=['simple']), key=attrgetter('name')) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def get_port(number, mock_stdout, index=None): if index is None: self.command.dispatch(['port', 'simple', str(number)], None) @@ -547,7 +547,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(containers), 1) self.assertIn("FOO=1", containers[0].get('Config.Env')) - @patch.dict(os.environ) + @mock.patch.dict(os.environ) def test_home_and_env_var_in_volume_path(self): os.environ['VOLUME_NAME'] = 'my-volume' os.environ['HOME'] = '/tmp/home-dir' diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 1d53465f63..fe54d4ae24 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -7,10 +7,10 @@ import tempfile from os import path from docker.errors import APIError -from mock import patch from six import StringIO from six import text_type +from .. import mock from .testcases import DockerClientTestCase from compose import __version__ from compose.const import LABEL_CONTAINER_NUMBER @@ -460,7 +460,7 @@ class ServiceTest(DockerClientTestCase): ) container = create_and_start_container(service) container.wait() - self.assertIn('success', container.logs()) + self.assertIn(b'success', container.logs()) self.assertEqual(len(self.client.images(name='composetest_test')), 1) def test_start_container_uses_tagged_image_if_it_exists(self): @@ -473,7 +473,7 @@ class ServiceTest(DockerClientTestCase): ) container = create_and_start_container(service) container.wait() - self.assertIn('success', container.logs()) + self.assertIn(b'success', container.logs()) def test_start_container_creates_ports(self): service = self.create_service('web', ports=[8000]) @@ -581,7 +581,7 @@ class ServiceTest(DockerClientTestCase): service.scale(0) self.assertEqual(len(service.containers()), 0) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_scale_with_stopped_containers(self, mock_stdout): """ Given there are some stopped containers and scale is called with a @@ -608,7 +608,7 @@ class ServiceTest(DockerClientTestCase): self.assertNotIn('Creating', captured_output) self.assertIn('Starting', captured_output) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_scale_with_stopped_containers_and_needing_creation(self, mock_stdout): """ Given there are some stopped containers and scale is called with a @@ -632,7 +632,7 @@ class ServiceTest(DockerClientTestCase): self.assertIn('Creating', captured_output) self.assertIn('Starting', captured_output) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_scale_with_api_returns_errors(self, mock_stdout): """ Test that when scaling if the API returns an error, that error is handled @@ -642,7 +642,7 @@ class ServiceTest(DockerClientTestCase): next_number = service._next_container_number() service.create_container(number=next_number, quiet=True) - with patch( + with mock.patch( 'compose.container.Container.create', side_effect=APIError(message="testing", response={}, explanation="Boom")): @@ -652,7 +652,7 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(service.containers()[0].is_running) self.assertIn("ERROR: for 2 Boom", mock_stdout.getvalue()) - @patch('sys.stdout', new_callable=StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_scale_with_api_returns_unexpected_exception(self, mock_stdout): """ Test that when scaling if the API returns an error, that is not of type @@ -662,7 +662,7 @@ class ServiceTest(DockerClientTestCase): next_number = service._next_container_number() service.create_container(number=next_number, quiet=True) - with patch( + with mock.patch( 'compose.container.Container.create', side_effect=ValueError("BOOM")): with self.assertRaises(ValueError): @@ -671,7 +671,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(len(service.containers()), 1) self.assertTrue(service.containers()[0].is_running) - @patch('compose.service.log') + @mock.patch('compose.service.log') def test_scale_with_desired_number_already_achieved(self, mock_log): """ Test that calling scale with a desired number that is equal to the @@ -694,7 +694,7 @@ class ServiceTest(DockerClientTestCase): captured_output = mock_log.info.call_args[0] self.assertIn('Desired container number already achieved', captured_output) - @patch('compose.service.log') + @mock.patch('compose.service.log') def test_scale_with_custom_container_name_outputs_warning(self, mock_log): """ Test that calling scale on a service that has a custom container name @@ -815,7 +815,7 @@ class ServiceTest(DockerClientTestCase): for k, v in {'ONE': '1', 'TWO': '2', 'THREE': '3', 'FOO': 'baz', 'DOO': 'dah'}.items(): self.assertEqual(env[k], v) - @patch.dict(os.environ) + @mock.patch.dict(os.environ) def test_resolve_env(self): os.environ['FILE_DEF'] = 'E1' os.environ['FILE_DEF_EMPTY'] = 'E2' diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index 6c2dc5f81e..5ccde73ad3 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -3,9 +3,8 @@ from __future__ import unicode_literals import os -import mock - from compose.cli import docker_client +from tests import mock from tests import unittest diff --git a/tests/unit/cli/verbose_proxy_test.py b/tests/unit/cli/verbose_proxy_test.py index 6036974c6f..f77568dc08 100644 --- a/tests/unit/cli/verbose_proxy_test.py +++ b/tests/unit/cli/verbose_proxy_test.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +import six + from compose.cli import verbose_proxy from tests import unittest @@ -8,7 +10,8 @@ from tests import unittest class VerboseProxyTestCase(unittest.TestCase): def test_format_call(self): - expected = "(u'arg1', True, key=u'value')" + prefix = '' if six.PY3 else 'u' + expected = "(%(p)s'arg1', True, key=%(p)s'value')" % dict(p=prefix) actual = verbose_proxy.format_call( ("arg1", True), {'key': 'value'}) @@ -21,7 +24,7 @@ class VerboseProxyTestCase(unittest.TestCase): self.assertEqual(expected, actual) def test_format_return(self): - expected = "{u'Id': u'ok'}" + expected = repr({'Id': 'ok'}) actual = verbose_proxy.format_return({'Id': 'ok'}, 2) self.assertEqual(expected, actual) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 35be4e9264..7d22ad02ff 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -4,8 +4,8 @@ from __future__ import unicode_literals import os import docker -import mock +from .. import mock from .. import unittest from compose.cli.docopt_command import NoSuchCommand from compose.cli.errors import UserError diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 3d1a53214d..7ecb6c4a2d 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -1,9 +1,11 @@ +from __future__ import print_function + import os import shutil import tempfile +from operator import itemgetter -import mock - +from .. import mock from .. import unittest from compose.config import config from compose.config.errors import ConfigurationError @@ -30,7 +32,7 @@ class ConfigTest(unittest.TestCase): ) self.assertEqual( - sorted(service_dicts, key=lambda d: d['name']), + sorted(service_dicts, key=itemgetter('name')), sorted([ { 'name': 'bar', @@ -41,7 +43,7 @@ class ConfigTest(unittest.TestCase): 'name': 'foo', 'image': 'busybox', } - ], key=lambda d: d['name']) + ], key=itemgetter('name')) ) def test_load_throws_error_when_not_dict(self): @@ -885,24 +887,24 @@ class ExtendsTest(unittest.TestCase): other_config = {'web': {'links': ['db']}} with mock.patch.object(config, 'load_yaml', return_value=other_config): - print load_config() + print(load_config()) with self.assertRaisesRegexp(ConfigurationError, 'volumes_from'): other_config = {'web': {'volumes_from': ['db']}} with mock.patch.object(config, 'load_yaml', return_value=other_config): - print load_config() + print(load_config()) with self.assertRaisesRegexp(ConfigurationError, 'net'): other_config = {'web': {'net': 'container:db'}} with mock.patch.object(config, 'load_yaml', return_value=other_config): - print load_config() + print(load_config()) other_config = {'web': {'net': 'host'}} with mock.patch.object(config, 'load_yaml', return_value=other_config): - print load_config() + print(load_config()) def test_volume_path(self): dicts = load_from_filename('tests/fixtures/volume-path/docker-compose.yml') diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index e2381c7c27..1eba9f656d 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -1,8 +1,8 @@ from __future__ import unicode_literals import docker -import mock +from .. import mock from .. import unittest from compose.container import Container from compose.container import get_container_name diff --git a/tests/unit/log_printer_test.py b/tests/unit/log_printer_test.py index bfd16affe1..f3fa64c614 100644 --- a/tests/unit/log_printer_test.py +++ b/tests/unit/log_printer_test.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import os +import six + from .. import unittest from compose.cli.log_printer import LogPrinter @@ -30,16 +32,17 @@ class LogPrinterTest(unittest.TestCase): output = self.get_default_output() self.assertIn('\033[', output) + @unittest.skipIf(six.PY3, "Only test unicode in python2") def test_unicode(self): - glyph = u'\u2022'.encode('utf-8') + glyph = u'\u2022' def reader(*args, **kwargs): - yield glyph + b'\n' + yield glyph + '\n' container = MockContainer(reader) output = run_log_printer([container]) - self.assertIn(glyph, output) + self.assertIn(glyph, output.decode('utf-8')) def run_log_printer(containers, monochrome=False): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 7d633c9505..37ebe5148d 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -1,8 +1,8 @@ from __future__ import unicode_literals import docker -import mock +from .. import mock from .. import unittest from compose.const import LABEL_SERVICE from compose.container import Container diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 12bb4ac2d7..3bb3e1722b 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -2,9 +2,9 @@ from __future__ import absolute_import from __future__ import unicode_literals import docker -import mock from docker.utils import LogConfig +from .. import mock from .. import unittest from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT diff --git a/tests/unit/split_buffer_test.py b/tests/unit/split_buffer_test.py index efd99411ae..1164609937 100644 --- a/tests/unit/split_buffer_test.py +++ b/tests/unit/split_buffer_test.py @@ -8,38 +8,38 @@ from compose.cli.utils import split_buffer class SplitBufferTest(unittest.TestCase): def test_single_line_chunks(self): def reader(): - yield b'abc\n' - yield b'def\n' - yield b'ghi\n' + yield 'abc\n' + yield 'def\n' + yield 'ghi\n' - self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi\n']) + self.assert_produces(reader, ['abc\n', 'def\n', 'ghi\n']) def test_no_end_separator(self): def reader(): - yield b'abc\n' - yield b'def\n' - yield b'ghi' + yield 'abc\n' + yield 'def\n' + yield 'ghi' - self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi']) + self.assert_produces(reader, ['abc\n', 'def\n', 'ghi']) def test_multiple_line_chunk(self): def reader(): - yield b'abc\ndef\nghi' + yield 'abc\ndef\nghi' - self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi']) + self.assert_produces(reader, ['abc\n', 'def\n', 'ghi']) def test_chunked_line(self): def reader(): - yield b'a' - yield b'b' - yield b'c' - yield b'\n' - yield b'd' + yield 'a' + yield 'b' + yield 'c' + yield '\n' + yield 'd' - self.assert_produces(reader, [b'abc\n', b'd']) + self.assert_produces(reader, ['abc\n', 'd']) def test_preserves_unicode_sequences_within_lines(self): - string = u"a\u2022c\n".encode('utf-8') + string = u"a\u2022c\n" def reader(): yield string @@ -47,7 +47,7 @@ class SplitBufferTest(unittest.TestCase): self.assert_produces(reader, [string]) def assert_produces(self, reader, expectations): - split = split_buffer(reader(), b'\n') + split = split_buffer(reader(), '\n') for (actual, expected) in zip(split, expectations): self.assertEqual(type(actual), type(expected)) diff --git a/tox.ini b/tox.ini index 3a69c5784c..35523a9697 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,pre-commit +envlist = py27,py34,pre-commit [testenv] usedevelop=True @@ -7,7 +7,6 @@ passenv = LD_LIBRARY_PATH deps = -rrequirements.txt - -rrequirements-dev.txt commands = nosetests -v {posargs} flake8 compose tests setup.py @@ -20,6 +19,26 @@ commands = pre-commit install pre-commit run --all-files +[testenv:py26] +deps = + {[testenv]deps} + -rrequirements-dev-py2.txt + +[testenv:py27] +deps = {[testenv:py26]deps} + +[testenv:pypy] +deps = {[testenv:py26]deps} + +[testenv:py33] +deps = + {[testenv]deps} + -rrequirements-dev-py3.txt + +[testenv:py34] +deps = {[testenv:py33]deps} + + [flake8] # ignore line-length for now ignore = E501,E203 From 9aa61e596e2475fa0bbcf227f2c388f6a9df471a Mon Sep 17 00:00:00 2001 From: funkyfuture Date: Thu, 26 Mar 2015 23:28:02 +0100 Subject: [PATCH 2/8] Run tests against Python 2.6, 2.7, 3.3, 3.4 and PyPy2 In particular it includes: - some extension of CONTRIBUTING.md - one fix for Python 2.6 in tests/integration/cli_test.py - one fix for Python 3.3 in tests/integration/service_test.py - removal of unused imports Make stream_output Python 3-compatible Signed-off-by: Frank Sachsenheim --- Dockerfile | 4 ++-- compose/container.py | 7 +++---- compose/progress_stream.py | 2 ++ compose/project.py | 2 +- requirements-dev.txt | 2 ++ script/test-versions | 2 +- setup.py | 2 +- tests/integration/cli_test.py | 2 +- tests/integration/service_test.py | 2 +- tests/unit/progress_stream_test.py | 1 - tox.ini | 3 ++- 11 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 requirements-dev.txt diff --git a/Dockerfile b/Dockerfile index 1986ac5a5b..a4cc99fea0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,8 +66,8 @@ WORKDIR /code/ ADD requirements.txt /code/ RUN pip install -r requirements.txt -ADD requirements-dev-py2.txt /code/ -RUN pip install -r requirements-dev-py2.txt +ADD requirements-dev.txt /code/ +RUN pip install -r requirements-dev.txt RUN pip install tox==2.1.1 diff --git a/compose/container.py b/compose/container.py index f727c8673a..6f426532ac 100644 --- a/compose/container.py +++ b/compose/container.py @@ -1,9 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals -from functools import reduce - -import six +from six import iteritems +from six.moves import reduce from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_SERVICE @@ -90,7 +89,7 @@ class Container(object): private=private, **public[0]) return ', '.join(format_port(*item) - for item in sorted(six.iteritems(self.ports))) + for item in sorted(iteritems(self.ports))) @property def labels(self): diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 582c09fb9e..e2300fd4af 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -17,6 +17,8 @@ def stream_output(output, stream): diff = 0 for chunk in output: + if six.PY3 and not isinstance(chunk, str): + chunk = chunk.decode('utf-8') event = json.loads(chunk) all_events.append(event) diff --git a/compose/project.py b/compose/project.py index d14941e72b..cd88b2988b 100644 --- a/compose/project.py +++ b/compose/project.py @@ -2,9 +2,9 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging -from functools import reduce from docker.errors import APIError +from six.moves import reduce from .config import ConfigurationError from .config import get_service_name_from_net diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000000..cc98422530 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +flake8 +tox diff --git a/script/test-versions b/script/test-versions index d67a6f5e12..e2102e449c 100755 --- a/script/test-versions +++ b/script/test-versions @@ -24,5 +24,5 @@ for version in $DOCKER_VERSIONS; do -e "DOCKER_DAEMON_ARGS" \ --entrypoint="script/dind" \ "$TAG" \ - script/wrapdocker nosetests --with-coverage --cover-branches --cover-package=compose --cover-erase --cover-html-dir=coverage-html --cover-html "$@" + script/wrapdocker tox "$@" done diff --git a/setup.py b/setup.py index b7fd440348..cdb5686cf3 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ tests_require = [ ] -if sys.version_info < (2, 6): +if sys.version_info < (2, 7): tests_require.append('unittest2') if sys.version_info[:1] < (3,): tests_require.append('pyinstaller') diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 609370a3e1..9552bf6a66 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -275,7 +275,7 @@ class CLITestCase(DockerClientTestCase): self.command.base_dir = 'tests/fixtures/user-composefile' name = 'service' user = 'sshd' - args = ['run', '--user={}'.format(user), name] + args = ['run', '--user={user}'.format(user=user), name] self.command.dispatch(args, None) service = self.project.get_service(name) container = service.containers(stopped=True, one_off=True)[0] diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index fe54d4ae24..effd356dfa 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -358,7 +358,7 @@ class ServiceTest(DockerClientTestCase): ) old_container = create_and_start_container(service) - self.assertEqual(old_container.get('Volumes').keys(), ['/data']) + self.assertEqual(list(old_container.get('Volumes').keys()), ['/data']) volume_path = old_container.get('Volumes')['/data'] new_container, = service.execute_convergence_plan( diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index 5674f4e4e8..e38a744353 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -8,7 +8,6 @@ from tests import unittest class ProgressStreamTestCase(unittest.TestCase): - def test_stream_output(self): output = [ '{"status": "Downloading", "progressDetail": {"current": ' diff --git a/tox.ini b/tox.ini index 35523a9697..2e3edd2a50 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ passenv = deps = -rrequirements.txt commands = - nosetests -v {posargs} + nosetests -v --with-coverage --cover-branches --cover-package=compose --cover-erase --cover-html-dir=coverage-html --cover-html {posargs} flake8 compose tests setup.py [testenv:pre-commit] @@ -38,6 +38,7 @@ deps = [testenv:py34] deps = {[testenv:py33]deps} +# TODO pypy3 [flake8] # ignore line-length for now From 2943ac6812bcc8cdcd5b877155cdf69dd08c5b8a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 2 Jul 2015 22:35:20 -0400 Subject: [PATCH 3/8] Cleanup requirements.txt so we don't have to maintain separate copies for py2 and py3. Signed-off-by: Daniel Nephin --- Dockerfile | 14 ++++++++++++++ MANIFEST.in | 2 +- compose/container.py | 7 ++++--- compose/project.py | 4 ++-- compose/service.py | 8 +++++--- compose/utils.py | 2 +- requirements-dev-py2.txt | 7 ------- requirements-dev-py3.txt | 2 -- requirements-dev.txt | 7 +++++-- script/test-versions | 2 +- setup.py | 3 --- tests/integration/resilience_test.py | 3 +-- tests/integration/service_test.py | 4 ++-- tests/unit/service_test.py | 2 +- tox.ini | 22 +++++++--------------- 15 files changed, 44 insertions(+), 45 deletions(-) delete mode 100644 requirements-dev-py2.txt delete mode 100644 requirements-dev-py3.txt diff --git a/Dockerfile b/Dockerfile index a4cc99fea0..546e28d69d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,18 @@ RUN set -ex; \ rm -rf /Python-2.7.9; \ rm Python-2.7.9.tgz +# Build python 3.4 from source +RUN set -ex; \ + curl -LO https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tgz; \ + tar -xzf Python-3.4.3.tgz; \ + cd Python-3.4.3; \ + ./configure --enable-shared; \ + make; \ + make install; \ + cd ..; \ + rm -rf /Python-3.4.3; \ + rm Python-3.4.3.tgz + # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib @@ -63,6 +75,8 @@ RUN ln -s /usr/local/bin/docker-1.7.1 /usr/local/bin/docker RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ +RUN pip install tox + ADD requirements.txt /code/ RUN pip install -r requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index 7420485961..7d48d347a8 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,7 @@ include Dockerfile include LICENSE include requirements.txt -include requirements-dev*.txt +include requirements-dev.txt include tox.ini include *.md include compose/config/schema.json diff --git a/compose/container.py b/compose/container.py index 6f426532ac..f727c8673a 100644 --- a/compose/container.py +++ b/compose/container.py @@ -1,8 +1,9 @@ from __future__ import absolute_import from __future__ import unicode_literals -from six import iteritems -from six.moves import reduce +from functools import reduce + +import six from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_SERVICE @@ -89,7 +90,7 @@ class Container(object): private=private, **public[0]) return ', '.join(format_port(*item) - for item in sorted(iteritems(self.ports))) + for item in sorted(six.iteritems(self.ports))) @property def labels(self): diff --git a/compose/project.py b/compose/project.py index cd88b2988b..a3127c6c29 100644 --- a/compose/project.py +++ b/compose/project.py @@ -2,9 +2,9 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging +from functools import reduce from docker.errors import APIError -from six.moves import reduce from .config import ConfigurationError from .config import get_service_name_from_net @@ -340,7 +340,7 @@ class Project(object): self.service_names, ) - return filter(matches_service_names, containers) + return [c for c in containers if matches_service_names(c)] def _inject_deps(self, acc, service): dep_names = service.get_dependency_names() diff --git a/compose/service.py b/compose/service.py index 647516ba84..d8a26e73a9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -709,7 +709,9 @@ class Service(object): def build(self, no_cache=False): log.info('Building %s...' % self.name) - path = six.binary_type(self.options['build']) + path = self.options['build'] + if not six.PY3: + path = path.encode('utf8') build_output = self.client.build( path=path, @@ -840,7 +842,7 @@ def merge_volume_bindings(volumes_option, previous_container): volume_bindings.update( get_container_data_volumes(previous_container, volumes_option)) - return volume_bindings.values() + return list(volume_bindings.values()) def get_container_data_volumes(container, volumes_option): @@ -853,7 +855,7 @@ def get_container_data_volumes(container, volumes_option): container_volumes = container.get('Volumes') or {} image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {} - for volume in set(volumes_option + image_volumes.keys()): + for volume in set(volumes_option + list(image_volumes)): volume = parse_volume_spec(volume) # No need to preserve host volumes if volume.external: diff --git a/compose/utils.py b/compose/utils.py index bd8922670e..0cbefba9be 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -97,5 +97,5 @@ def write_out_msg(stream, lines, msg_index, msg, status="done"): def json_hash(obj): dump = json.dumps(obj, sort_keys=True, separators=(',', ':')) h = hashlib.sha256() - h.update(dump) + h.update(dump.encode('utf8')) return h.hexdigest() diff --git a/requirements-dev-py2.txt b/requirements-dev-py2.txt deleted file mode 100644 index 97fc4fed86..0000000000 --- a/requirements-dev-py2.txt +++ /dev/null @@ -1,7 +0,0 @@ -coverage==3.7.1 -flake8==2.3.0 -git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller -mock >= 1.0.1 -nose==1.3.4 -pep8==1.6.1 -unittest2==0.8.0 diff --git a/requirements-dev-py3.txt b/requirements-dev-py3.txt deleted file mode 100644 index a2ba1c8b45..0000000000 --- a/requirements-dev-py3.txt +++ /dev/null @@ -1,2 +0,0 @@ -flake8 -nose >= 1.3.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index cc98422530..9e830733c1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,5 @@ -flake8 -tox +flake8==2.3.0 +git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller +mock >= 1.0.1 +nose==1.3.4 +pep8==1.6.1 diff --git a/script/test-versions b/script/test-versions index e2102e449c..f39c17e819 100755 --- a/script/test-versions +++ b/script/test-versions @@ -24,5 +24,5 @@ for version in $DOCKER_VERSIONS; do -e "DOCKER_DAEMON_ARGS" \ --entrypoint="script/dind" \ "$TAG" \ - script/wrapdocker tox "$@" + script/wrapdocker tox -e py27,py34 -- "$@" done diff --git a/setup.py b/setup.py index cdb5686cf3..33335047bc 100644 --- a/setup.py +++ b/setup.py @@ -41,9 +41,7 @@ install_requires = [ tests_require = [ - 'mock >= 1.0.1', 'nose', - 'pyinstaller', 'flake8', ] @@ -51,7 +49,6 @@ tests_require = [ if sys.version_info < (2, 7): tests_require.append('unittest2') if sys.version_info[:1] < (3,): - tests_require.append('pyinstaller') tests_require.append('mock >= 1.0.1') diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index b1faf99dff..82a4680d8b 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -1,8 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals -import mock - +from .. import mock from .testcases import DockerClientTestCase from compose.project import Project diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index effd356dfa..f300c6d531 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -363,7 +363,7 @@ class ServiceTest(DockerClientTestCase): new_container, = service.execute_convergence_plan( ConvergencePlan('recreate', [old_container])) - self.assertEqual(new_container.get('Volumes').keys(), ['/data']) + self.assertEqual(list(new_container.get('Volumes')), ['/data']) self.assertEqual(new_container.get('Volumes')['/data'], volume_path) def test_start_container_passes_through_options(self): @@ -498,7 +498,7 @@ class ServiceTest(DockerClientTestCase): with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: f.write("FROM busybox\n") - with open(os.path.join(base_dir, b'foo\xE2bar'), 'w') as f: + with open(os.path.join(base_dir.encode('utf8'), b'foo\xE2bar'), 'w') as f: f.write("hello world\n") self.create_service('web', build=text_type(base_dir)).build() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 3bb3e1722b..f224752770 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -41,7 +41,7 @@ class ServiceTest(unittest.TestCase): dict(Name=str(i), Image='foo', Id=i) for i in range(3) ] service = Service('db', self.mock_client, 'myproject', image='foo') - self.assertEqual([c.id for c in service.containers()], range(3)) + self.assertEqual([c.id for c in service.containers()], list(range(3))) expected_labels = [ '{0}=myproject'.format(LABEL_PROJECT), diff --git a/tox.ini b/tox.ini index 2e3edd2a50..a2bd6b6b9b 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,8 @@ envlist = py27,py34,pre-commit usedevelop=True passenv = LD_LIBRARY_PATH +setenv = + HOME=/tmp deps = -rrequirements.txt commands = @@ -19,26 +21,16 @@ commands = pre-commit install pre-commit run --all-files -[testenv:py26] -deps = - {[testenv]deps} - -rrequirements-dev-py2.txt - [testenv:py27] -deps = {[testenv:py26]deps} - -[testenv:pypy] -deps = {[testenv:py26]deps} - -[testenv:py33] deps = {[testenv]deps} - -rrequirements-dev-py3.txt + -rrequirements-dev.txt [testenv:py34] -deps = {[testenv:py33]deps} - -# TODO pypy3 +deps = + {[testenv]deps} + flake8 + nose [flake8] # ignore line-length for now From feaa4a5f1aa97caf984d08e50d4e6c384fe1f0ae Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 11:51:38 -0400 Subject: [PATCH 4/8] Unit tests passing again. Signed-off-by: Daniel Nephin --- compose/config/validation.py | 2 +- compose/service.py | 4 ++-- compose/utils.py | 4 ++-- tests/unit/config_test.py | 27 +++++++++++++-------------- tests/unit/service_test.py | 2 +- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 8911f5ae1d..e5f195f400 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -150,7 +150,7 @@ def process_errors(errors): config_key = error.path[0] required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, _clean_error_message(error.message))) elif error.validator == 'dependencies': - dependency_key = error.validator_value.keys()[0] + dependency_key = list(error.validator_value.keys())[0] required_keys = ",".join(error.validator_value[dependency_key]) required.append("Invalid '{}' configuration for '{}' service: when defining '{}' you must set '{}' as well".format( dependency_key, service_name, dependency_key, required_keys)) diff --git a/compose/service.py b/compose/service.py index d8a26e73a9..9c0bc44391 100644 --- a/compose/service.py +++ b/compose/service.py @@ -103,11 +103,11 @@ class Service(object): def containers(self, stopped=False, one_off=False, filters={}): filters.update({'label': self.labels(one_off=one_off)}) - containers = filter(None, [ + containers = list(filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, - filters=filters)]) + filters=filters)])) if not containers: check_for_legacy_containers( diff --git a/compose/utils.py b/compose/utils.py index 0cbefba9be..738fcacaff 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -3,11 +3,11 @@ import hashlib import json import logging import sys -from Queue import Empty -from Queue import Queue from threading import Thread from docker.errors import APIError +from six.moves.queue import Empty +from six.moves.queue import Queue log = logging.getLogger(__name__) diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index 7ecb6c4a2d..ccd5b57bf7 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -18,6 +18,10 @@ def make_service_dict(name, service_dict, working_dir): return config.ServiceLoader(working_dir=working_dir).make_service_dict(name, service_dict) +def service_sort(services): + return sorted(services, key=itemgetter('name')) + + class ConfigTest(unittest.TestCase): def test_load(self): service_dicts = config.load( @@ -32,8 +36,8 @@ class ConfigTest(unittest.TestCase): ) self.assertEqual( - sorted(service_dicts, key=itemgetter('name')), - sorted([ + service_sort(service_dicts), + service_sort([ { 'name': 'bar', 'image': 'busybox', @@ -43,7 +47,7 @@ class ConfigTest(unittest.TestCase): 'name': 'foo', 'image': 'busybox', } - ], key=itemgetter('name')) + ]) ) def test_load_throws_error_when_not_dict(self): @@ -684,12 +688,7 @@ class ExtendsTest(unittest.TestCase): def test_extends(self): service_dicts = load_from_filename('tests/fixtures/extends/docker-compose.yml') - service_dicts = sorted( - service_dicts, - key=lambda sd: sd['name'], - ) - - self.assertEqual(service_dicts, [ + self.assertEqual(service_sort(service_dicts), service_sort([ { 'name': 'mydb', 'image': 'busybox', @@ -706,7 +705,7 @@ class ExtendsTest(unittest.TestCase): "BAZ": "2", }, } - ]) + ])) def test_nested(self): service_dicts = load_from_filename('tests/fixtures/extends/nested.yml') @@ -728,7 +727,7 @@ class ExtendsTest(unittest.TestCase): We specify a 'file' key that is the filename we're already in. """ service_dicts = load_from_filename('tests/fixtures/extends/specify-file-as-self.yml') - self.assertEqual(service_dicts, [ + self.assertEqual(service_sort(service_dicts), service_sort([ { 'environment': { @@ -749,7 +748,7 @@ class ExtendsTest(unittest.TestCase): 'image': 'busybox', 'name': 'web' } - ]) + ])) def test_circular(self): try: @@ -856,7 +855,7 @@ class ExtendsTest(unittest.TestCase): config is valid and correctly extends from itself. """ service_dicts = load_from_filename('tests/fixtures/extends/no-file-specified.yml') - self.assertEqual(service_dicts, [ + self.assertEqual(service_sort(service_dicts), service_sort([ { 'name': 'myweb', 'image': 'busybox', @@ -872,7 +871,7 @@ class ExtendsTest(unittest.TestCase): "BAZ": "3", } } - ]) + ])) def test_blacklisted_options(self): def load_config(): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index f224752770..4708616e34 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -34,7 +34,7 @@ class ServiceTest(unittest.TestCase): def test_containers(self): service = Service('db', self.mock_client, 'myproject', image='foo') self.mock_client.containers.return_value = [] - self.assertEqual(service.containers(), []) + self.assertEqual(list(service.containers()), []) def test_containers_with_containers(self): self.mock_client.containers.return_value = [ From 7e4c3142d721ccab37ee6e34d93e9214fc3b89ef Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 12:43:08 -0400 Subject: [PATCH 5/8] Have log_printer use utf8 stream. Signed-off-by: Daniel Nephin --- compose/cli/log_printer.py | 6 +++--- tests/unit/log_printer_test.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index c7d0b638f8..034551ec62 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -5,7 +5,9 @@ import sys from itertools import cycle import six +from six import next +from compose import utils from . import colors from .multiplexer import Multiplexer from .utils import split_buffer @@ -17,13 +19,11 @@ class LogPrinter(object): self.attach_params = attach_params or {} self.prefix_width = self._calculate_prefix_width(containers) self.generators = self._make_log_generators(monochrome) - self.output = output + self.output = utils.get_output_stream(output) def run(self): mux = Multiplexer(self.generators) for line in mux.loop(): - if isinstance(line, six.text_type) and not six.PY3: - line = line.encode('utf-8') self.output.write(line) def _calculate_prefix_width(self, containers): diff --git a/tests/unit/log_printer_test.py b/tests/unit/log_printer_test.py index f3fa64c614..284934a6af 100644 --- a/tests/unit/log_printer_test.py +++ b/tests/unit/log_printer_test.py @@ -32,7 +32,6 @@ class LogPrinterTest(unittest.TestCase): output = self.get_default_output() self.assertIn('\033[', output) - @unittest.skipIf(six.PY3, "Only test unicode in python2") def test_unicode(self): glyph = u'\u2022' @@ -42,7 +41,10 @@ class LogPrinterTest(unittest.TestCase): container = MockContainer(reader) output = run_log_printer([container]) - self.assertIn(glyph, output.decode('utf-8')) + if six.PY2: + output = output.decode('utf-8') + + self.assertIn(glyph, output) def run_log_printer(containers, monochrome=False): From 71ff872e8e6f09a15f39f90b7faba2b44201c46d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 13:16:13 -0400 Subject: [PATCH 6/8] Update unit tests for stream_output to match the behaviour of a docker-py response. Signed-off-by: Daniel Nephin --- compose/cli/log_printer.py | 3 +-- compose/progress_stream.py | 8 ++++---- compose/project.py | 4 ++-- compose/service.py | 2 ++ compose/utils.py | 9 ++++++++- tests/integration/legacy_test.py | 4 ++-- tests/unit/progress_stream_test.py | 18 +++++++++--------- tests/unit/service_test.py | 2 +- 8 files changed, 29 insertions(+), 21 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 034551ec62..69ada850e5 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -4,13 +4,12 @@ from __future__ import unicode_literals import sys from itertools import cycle -import six from six import next -from compose import utils from . import colors from .multiplexer import Multiplexer from .utils import split_buffer +from compose import utils class LogPrinter(object): diff --git a/compose/progress_stream.py b/compose/progress_stream.py index e2300fd4af..c44b33e561 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -1,8 +1,9 @@ -import codecs import json import six +from compose import utils + class StreamOutputError(Exception): pass @@ -10,14 +11,13 @@ class StreamOutputError(Exception): def stream_output(output, stream): is_terminal = hasattr(stream, 'isatty') and stream.isatty() - if not six.PY3: - stream = codecs.getwriter('utf-8')(stream) + stream = utils.get_output_stream(stream) all_events = [] lines = {} diff = 0 for chunk in output: - if six.PY3 and not isinstance(chunk, str): + if six.PY3: chunk = chunk.decode('utf-8') event = json.loads(chunk) all_events.append(event) diff --git a/compose/project.py b/compose/project.py index a3127c6c29..542c8785e5 100644 --- a/compose/project.py +++ b/compose/project.py @@ -324,11 +324,11 @@ class Project(object): else: service_names = self.service_names - containers = filter(None, [ + containers = list(filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, - filters={'label': self.labels(one_off=one_off)})]) + filters={'label': self.labels(one_off=one_off)})])) def matches_service_names(container): return container.labels.get(LABEL_SERVICE) in service_names diff --git a/compose/service.py b/compose/service.py index 9c0bc44391..a15ee1b9af 100644 --- a/compose/service.py +++ b/compose/service.py @@ -710,6 +710,8 @@ class Service(object): log.info('Building %s...' % self.name) path = self.options['build'] + # python2 os.path() doesn't support unicode, so we need to encode it to + # a byte string if not six.PY3: path = path.encode('utf8') diff --git a/compose/utils.py b/compose/utils.py index 738fcacaff..c729228409 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -5,6 +5,7 @@ import logging import sys from threading import Thread +import six from docker.errors import APIError from six.moves.queue import Empty from six.moves.queue import Queue @@ -18,7 +19,7 @@ def parallel_execute(objects, obj_callable, msg_index, msg): For a given list of objects, call the callable passing in the first object we give it. """ - stream = codecs.getwriter('utf-8')(sys.stdout) + stream = get_output_stream() lines = [] errors = {} @@ -70,6 +71,12 @@ def parallel_execute(objects, obj_callable, msg_index, msg): stream.write("ERROR: for {} {} \n".format(error, errors[error])) +def get_output_stream(stream=sys.stdout): + if six.PY3: + return stream + return codecs.getwriter('utf-8')(stream) + + def write_out_msg(stream, lines, msg_index, msg, status="done"): """ Using special ANSI code characters we can write out the msg over the top of diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py index fa983e6d59..3465d57f49 100644 --- a/tests/integration/legacy_test.py +++ b/tests/integration/legacy_test.py @@ -1,8 +1,8 @@ import unittest from docker.errors import APIError -from mock import Mock +from .. import mock from .testcases import DockerClientTestCase from compose import legacy from compose.project import Project @@ -66,7 +66,7 @@ class UtilitiesTestCase(unittest.TestCase): ) def test_get_legacy_containers(self): - client = Mock() + client = mock.Mock() client.containers.return_value = [ { "Id": "abc123", diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index e38a744353..d8f7ec8363 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -10,27 +10,27 @@ from tests import unittest class ProgressStreamTestCase(unittest.TestCase): def test_stream_output(self): output = [ - '{"status": "Downloading", "progressDetail": {"current": ' - '31019763, "start": 1413653874, "total": 62763875}, ' - '"progress": "..."}', + b'{"status": "Downloading", "progressDetail": {"current": ' + b'31019763, "start": 1413653874, "total": 62763875}, ' + b'"progress": "..."}', ] events = progress_stream.stream_output(output, StringIO()) self.assertEqual(len(events), 1) def test_stream_output_div_zero(self): output = [ - '{"status": "Downloading", "progressDetail": {"current": ' - '0, "start": 1413653874, "total": 0}, ' - '"progress": "..."}', + b'{"status": "Downloading", "progressDetail": {"current": ' + b'0, "start": 1413653874, "total": 0}, ' + b'"progress": "..."}', ] events = progress_stream.stream_output(output, StringIO()) self.assertEqual(len(events), 1) def test_stream_output_null_total(self): output = [ - '{"status": "Downloading", "progressDetail": {"current": ' - '0, "start": 1413653874, "total": null}, ' - '"progress": "..."}', + b'{"status": "Downloading", "progressDetail": {"current": ' + b'0, "start": 1413653874, "total": null}, ' + b'"progress": "..."}', ] events = progress_stream.stream_output(output, StringIO()) self.assertEqual(len(events), 1) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 4708616e34..275bde1bdd 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -280,7 +280,7 @@ class ServiceTest(unittest.TestCase): def test_build_does_not_pull(self): self.mock_client.build.return_value = [ - '{"stream": "Successfully built 12345"}', + b'{"stream": "Successfully built 12345"}', ] service = Service('foo', client=self.mock_client, build='.') From bd7c032a00b7701e5b29a983bb5a83b202dcd952 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 14:49:51 -0400 Subject: [PATCH 7/8] Fix service integration tests. Signed-off-by: Daniel Nephin --- compose/utils.py | 4 ++-- requirements-dev.txt | 1 + tests/integration/service_test.py | 16 +++++----------- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/compose/utils.py b/compose/utils.py index c729228409..30284f97bd 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -19,7 +19,7 @@ def parallel_execute(objects, obj_callable, msg_index, msg): For a given list of objects, call the callable passing in the first object we give it. """ - stream = get_output_stream() + stream = get_output_stream(sys.stdout) lines = [] errors = {} @@ -71,7 +71,7 @@ def parallel_execute(objects, obj_callable, msg_index, msg): stream.write("ERROR: for {} {} \n".format(error, errors[error])) -def get_output_stream(stream=sys.stdout): +def get_output_stream(stream): if six.PY3: return stream return codecs.getwriter('utf-8')(stream) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9e830733c1..c8a694ab27 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c mock >= 1.0.1 nose==1.3.4 pep8==1.6.1 +coverage==3.7.1 diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index f300c6d531..bc9dcc6925 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -581,8 +581,7 @@ class ServiceTest(DockerClientTestCase): service.scale(0) self.assertEqual(len(service.containers()), 0) - @mock.patch('sys.stdout', new_callable=StringIO) - def test_scale_with_stopped_containers(self, mock_stdout): + def test_scale_with_stopped_containers(self): """ Given there are some stopped containers and scale is called with a desired number that is the same as the number of stopped containers, @@ -591,15 +590,11 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('web') next_number = service._next_container_number() valid_numbers = [next_number, next_number + 1] - service.create_container(number=next_number, quiet=True) - service.create_container(number=next_number + 1, quiet=True) + service.create_container(number=next_number) + service.create_container(number=next_number + 1) - for container in service.containers(): - self.assertFalse(container.is_running) - - service.scale(2) - - self.assertEqual(len(service.containers()), 2) + with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + service.scale(2) for container in service.containers(): self.assertTrue(container.is_running) self.assertTrue(container.number in valid_numbers) @@ -701,7 +696,6 @@ class ServiceTest(DockerClientTestCase): results in warning output. """ service = self.create_service('web', container_name='custom-container') - self.assertEqual(service.custom_container_name(), 'custom-container') service.scale(3) From 1451a6e1889b48a780759c41779e99b09ad16d18 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 24 Aug 2015 18:54:59 -0400 Subject: [PATCH 8/8] Python3 requires a locale Signed-off-by: Daniel Nephin --- Dockerfile | 5 +++++ requirements-dev.txt | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 546e28d69d..a9892031b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ FROM debian:wheezy RUN set -ex; \ apt-get update -qq; \ apt-get install -y \ + locales \ gcc \ make \ zlib1g \ @@ -61,6 +62,10 @@ RUN set -ex; \ rm -rf pip-7.0.1; \ rm pip-7.0.1.tar.gz +# Python3 requires a valid locale +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 + ENV ALL_DOCKER_VERSIONS 1.7.1 1.8.1 RUN set -ex; \ diff --git a/requirements-dev.txt b/requirements-dev.txt index c8a694ab27..adb4387df2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ +coverage==3.7.1 flake8==2.3.0 git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller mock >= 1.0.1 nose==1.3.4 pep8==1.6.1 -coverage==3.7.1