From e517061010835af60c4dc29008f4b734d31cbaeb Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 29 May 2014 11:18:51 +0100 Subject: [PATCH 1/7] Add /venv to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1623c04f54..1046518b72 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ /build /dist /docs/_site +/venv fig.spec From cfcabce593af61c328b4817ec40a7854be4148fb Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 29 May 2014 11:19:30 +0100 Subject: [PATCH 2/7] Extract stream_output to module --- fig/progress_stream.py | 81 ++++++++++++++++++++++++++++++++++++++++++ fig/service.py | 80 +---------------------------------------- 2 files changed, 82 insertions(+), 79 deletions(-) create mode 100644 fig/progress_stream.py diff --git a/fig/progress_stream.py b/fig/progress_stream.py new file mode 100644 index 0000000000..b0160f04fa --- /dev/null +++ b/fig/progress_stream.py @@ -0,0 +1,81 @@ +import json +import os + + +class StreamOutputError(Exception): + pass + + +def stream_output(output, stream): + is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno()) + all_events = [] + lines = {} + diff = 0 + + for chunk in output: + event = json.loads(chunk) + all_events.append(event) + + if 'progress' in event or 'progressDetail' in event: + image_id = event['id'] + + if image_id in lines: + diff = len(lines) - lines[image_id] + else: + lines[image_id] = len(lines) + stream.write("\n") + diff = 0 + + if is_terminal: + # move cursor up `diff` rows + stream.write("%c[%dA" % (27, diff)) + + print_output_event(event, stream, is_terminal) + + if 'id' in event and is_terminal: + # move cursor back down + stream.write("%c[%dB" % (27, diff)) + + stream.flush() + + return all_events + + +def print_output_event(event, stream, is_terminal): + if 'errorDetail' in event: + raise StreamOutputError(event['errorDetail']['message']) + + terminator = '' + + if is_terminal and 'stream' not in event: + # erase current line + stream.write("%c[2K\r" % 27) + terminator = "\r" + pass + elif 'progressDetail' in event: + return + + if 'time' in event: + stream.write("[%s] " % event['time']) + + if 'id' in event: + stream.write("%s: " % event['id']) + + if 'from' in event: + stream.write("(from %s) " % event['from']) + + status = event.get('status', '') + + if 'progress' in event: + stream.write("%s %s%s" % (status, event['progress'], terminator)) + elif 'progressDetail' in event: + detail = event['progressDetail'] + if 'current' in detail: + percentage = float(detail['current']) / float(detail['total']) * 100 + stream.write('%s (%.1f%%)%s' % (status, percentage, terminator)) + else: + stream.write('%s%s' % (status, terminator)) + elif 'stream' in event: + stream.write("%s%s" % (event['stream'], terminator)) + else: + stream.write("%s%s\n" % (status, terminator)) diff --git a/fig/service.py b/fig/service.py index 21a3ea40d2..2fc7125d06 100644 --- a/fig/service.py +++ b/fig/service.py @@ -5,8 +5,8 @@ import logging import re import os import sys -import json from .container import Container +from .progress_stream import stream_output, StreamOutputError log = logging.getLogger(__name__) @@ -343,84 +343,6 @@ class Service(object): return True -class StreamOutputError(Exception): - pass - - -def stream_output(output, stream): - is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno()) - all_events = [] - lines = {} - diff = 0 - - for chunk in output: - event = json.loads(chunk) - all_events.append(event) - - if 'progress' in event or 'progressDetail' in event: - image_id = event['id'] - - if image_id in lines: - diff = len(lines) - lines[image_id] - else: - lines[image_id] = len(lines) - stream.write("\n") - diff = 0 - - if is_terminal: - # move cursor up `diff` rows - stream.write("%c[%dA" % (27, diff)) - - print_output_event(event, stream, is_terminal) - - if 'id' in event and is_terminal: - # move cursor back down - stream.write("%c[%dB" % (27, diff)) - - stream.flush() - - return all_events - -def print_output_event(event, stream, is_terminal): - if 'errorDetail' in event: - raise StreamOutputError(event['errorDetail']['message']) - - terminator = '' - - if is_terminal and 'stream' not in event: - # erase current line - stream.write("%c[2K\r" % 27) - terminator = "\r" - pass - elif 'progressDetail' in event: - return - - if 'time' in event: - stream.write("[%s] " % event['time']) - - if 'id' in event: - stream.write("%s: " % event['id']) - - if 'from' in event: - stream.write("(from %s) " % event['from']) - - status = event.get('status', '') - - if 'progress' in event: - stream.write("%s %s%s" % (status, event['progress'], terminator)) - elif 'progressDetail' in event: - detail = event['progressDetail'] - if 'current' in detail: - percentage = float(detail['current']) / float(detail['total']) * 100 - stream.write('%s (%.1f%%)%s' % (status, percentage, terminator)) - else: - stream.write('%s%s' % (status, terminator)) - elif 'stream' in event: - stream.write("%s%s" % (event['stream'], terminator)) - else: - stream.write("%s%s\n" % (status, terminator)) - - NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$') From c246897af130e97eaab120c467ae008e35230f76 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 29 May 2014 12:11:26 +0100 Subject: [PATCH 3/7] Pass script/test arguments through to nosetests --- script/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test b/script/test index 1ceefaad88..5f0f21f4ac 100755 --- a/script/test +++ b/script/test @@ -1,2 +1,2 @@ #!/bin/sh -nosetests +nosetests $@ From 9eb3697b40c40176f2dffcffd5120765b4428684 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 29 May 2014 11:51:15 +0100 Subject: [PATCH 4/7] Encode all progress stream output as UTF-8 Closes #231. --- fig/progress_stream.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fig/progress_stream.py b/fig/progress_stream.py index b0160f04fa..2121629b8d 100644 --- a/fig/progress_stream.py +++ b/fig/progress_stream.py @@ -1,5 +1,6 @@ import json import os +import codecs class StreamOutputError(Exception): @@ -8,6 +9,7 @@ class StreamOutputError(Exception): def stream_output(output, stream): is_terminal = hasattr(stream, 'fileno') and os.isatty(stream.fileno()) + stream = codecs.getwriter('utf-8')(stream) all_events = [] lines = {} diff = 0 From 262248d8a6bfc78ba0a54232991868baf084a19b Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 29 May 2014 12:11:08 +0100 Subject: [PATCH 5/7] Firm up tests for split_buffer --- tests/unit/split_buffer_test.py | 47 ++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/tests/unit/split_buffer_test.py b/tests/unit/split_buffer_test.py index a78e99a6e5..41dc50e49b 100644 --- a/tests/unit/split_buffer_test.py +++ b/tests/unit/split_buffer_test.py @@ -6,32 +6,47 @@ from .. import unittest class SplitBufferTest(unittest.TestCase): def test_single_line_chunks(self): def reader(): - yield "abc\n" - yield "def\n" - yield "ghi\n" + yield b'abc\n' + yield b'def\n' + yield b'ghi\n' - self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi\n"]) + self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi\n']) def test_no_end_separator(self): def reader(): - yield "abc\n" - yield "def\n" - yield "ghi" + yield b'abc\n' + yield b'def\n' + yield b'ghi' - self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi"]) + self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi']) def test_multiple_line_chunk(self): def reader(): - yield "abc\ndef\nghi" + yield b'abc\ndef\nghi' - self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi"]) + self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi']) def test_chunked_line(self): def reader(): - yield "a" - yield "b" - yield "c" - yield "\n" - yield "d" + yield b'a' + yield b'b' + yield b'c' + yield b'\n' + yield b'd' - self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "d"]) + self.assert_produces(reader, [b'abc\n', b'd']) + + def test_preserves_unicode_sequences_within_lines(self): + string = u"a\u2022c\n".encode('utf-8') + + def reader(): + yield string + + self.assert_produces(reader, [string]) + + def assert_produces(self, reader, expectations): + split = split_buffer(reader(), b'\n') + + for (actual, expected) in zip(split, expectations): + self.assertEqual(type(actual), type(expected)) + self.assertEqual(actual, expected) From d0b5bcf26ae83be151cc5fc841a80a15cc301f50 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 18 Jun 2014 14:50:57 +0100 Subject: [PATCH 6/7] Pass byte strings straight through LogPrinter --- fig/cli/log_printer.py | 9 +++--- script/test | 2 +- tests/unit/log_printer_test.py | 57 ++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 tests/unit/log_printer_test.py diff --git a/fig/cli/log_printer.py b/fig/cli/log_printer.py index e302aecb77..0dc419e14f 100644 --- a/fig/cli/log_printer.py +++ b/fig/cli/log_printer.py @@ -10,16 +10,17 @@ from .utils import split_buffer class LogPrinter(object): - def __init__(self, containers, attach_params=None): + def __init__(self, containers, attach_params=None, output=sys.stdout): self.containers = containers self.attach_params = attach_params or {} self.prefix_width = self._calculate_prefix_width(containers) self.generators = self._make_log_generators() + self.output = output def run(self): mux = Multiplexer(self.generators) for line in mux.loop(): - sys.stdout.write(line.encode(sys.__stdout__.encoding or 'utf-8')) + self.output.write(line) def _calculate_prefix_width(self, containers): """ @@ -45,12 +46,12 @@ class LogPrinter(object): return generators def _make_log_generator(self, container, color_fn): - prefix = color_fn(self._generate_prefix(container)) + prefix = color_fn(self._generate_prefix(container)).encode('utf-8') # Attach to container before log printer starts running line_generator = split_buffer(self._attach(container), '\n') for line in line_generator: - yield prefix + line.decode('utf-8') + yield prefix + line exit_code = container.wait() yield color_fn("%s exited with code %s\n" % (container.name, exit_code)) diff --git a/script/test b/script/test index 5f0f21f4ac..c2cc315dd6 100755 --- a/script/test +++ b/script/test @@ -1,2 +1,2 @@ #!/bin/sh -nosetests $@ +PYTHONIOENCODING=ascii nosetests $@ diff --git a/tests/unit/log_printer_test.py b/tests/unit/log_printer_test.py new file mode 100644 index 0000000000..da2a83275c --- /dev/null +++ b/tests/unit/log_printer_test.py @@ -0,0 +1,57 @@ +from __future__ import unicode_literals +from __future__ import absolute_import +import os + +from fig.cli.log_printer import LogPrinter +from .. import unittest + + +class LogPrinterTest(unittest.TestCase): + def test_single_container(self): + def reader(*args, **kwargs): + yield "hello\nworld" + + container = MockContainer(reader) + output = run_log_printer([container]) + + self.assertIn('hello', output) + self.assertIn('world', output) + + def test_unicode(self): + glyph = u'\u2022'.encode('utf-8') + + def reader(*args, **kwargs): + yield glyph + b'\n' + + container = MockContainer(reader) + output = run_log_printer([container]) + + self.assertIn(glyph, output) + + +def run_log_printer(containers): + r, w = os.pipe() + reader, writer = os.fdopen(r, 'r'), os.fdopen(w, 'w') + printer = LogPrinter(containers, output=writer) + printer.run() + writer.close() + return reader.read() + + +class MockContainer(object): + def __init__(self, reader): + self._reader = reader + + @property + def name(self): + return 'myapp_web_1' + + @property + def name_without_project(self): + return 'web_1' + + def attach(self, *args, **kwargs): + return self._reader() + + def wait(self, *args, **kwargs): + return 0 From 2bd6e3d0a553166992f74622159192f8e9ef48b9 Mon Sep 17 00:00:00 2001 From: Tobias Bradtke Date: Tue, 27 May 2014 18:22:18 +0200 Subject: [PATCH 7/7] Do not encode chunk, just write as is. --- fig/cli/socketclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fig/cli/socketclient.py b/fig/cli/socketclient.py index 518b7af453..6cc1f2c570 100644 --- a/fig/cli/socketclient.py +++ b/fig/cli/socketclient.py @@ -81,7 +81,7 @@ class SocketClient: chunk = socket.recv(4096) if chunk: - stream.write(chunk.encode(stream.encoding or 'utf-8')) + stream.write(chunk) stream.flush() else: break