From 10b3188214fc6716387339bb0146d8d901962e93 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 11 Sep 2015 20:18:45 -0400 Subject: [PATCH] Support multiple config files Signed-off-by: Daniel Nephin --- compose/cli/command.py | 22 +++++---- compose/config/config.py | 66 +++++++++++++++++---------- tests/unit/config_test.py | 96 +++++++++++++++++++++------------------ 3 files changed, 105 insertions(+), 79 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 70b129d29..2120ec4db 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -51,24 +51,26 @@ class Command(DocoptCommand): handler(None, command_options) return - if 'FIG_FILE' in os.environ: - log.warn('The FIG_FILE environment variable is deprecated.') - log.warn('Please use COMPOSE_FILE instead.') - - explicit_config_path = ( - options.get('--file') or - os.environ.get('COMPOSE_FILE') or - os.environ.get('FIG_FILE')) - project = get_project( self.base_dir, - explicit_config_path, + get_config_path(options.get('--file')), project_name=options.get('--project-name'), verbose=options.get('--verbose')) handler(project, command_options) +def get_config_path(file_option): + if file_option: + return file_option + + if 'FIG_FILE' in os.environ: + log.warn('The FIG_FILE environment variable is deprecated.') + log.warn('Please use COMPOSE_FILE instead.') + + return [os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE')] + + def get_client(verbose=False): client = docker_client() if verbose: diff --git a/compose/config/config.py b/compose/config/config.py index 840a28a1b..204f70b66 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -2,6 +2,7 @@ import logging import os import sys from collections import namedtuple +from functools import reduce import six import yaml @@ -88,18 +89,24 @@ PATH_START_CHARS = [ log = logging.getLogger(__name__) -ConfigDetails = namedtuple('ConfigDetails', 'config working_dir filename') +ConfigDetails = namedtuple('ConfigDetails', 'working_dir configs') + +ConfigFile = namedtuple('ConfigFile', 'filename config') -def find(base_dir, filename): - if filename == '-': - return ConfigDetails(yaml.safe_load(sys.stdin), os.getcwd(), None) +def find(base_dir, filenames): + if filenames == ['-']: + return ConfigDetails( + os.getcwd(), + [ConfigFile(None, yaml.safe_load(sys.stdin))]) - if filename: - filename = os.path.join(base_dir, filename) + if filenames: + filenames = [os.path.join(base_dir, f) for f in filenames] else: - filename = get_config_path(base_dir) - return ConfigDetails(load_yaml(filename), os.path.dirname(filename), filename) + filenames = [get_config_path(base_dir)] + return ConfigDetails( + os.path.dirname(filenames[0]), + [ConfigFile(f, load_yaml(f)) for f in filenames]) def get_config_path(base_dir): @@ -133,29 +140,40 @@ def pre_process_config(config): Pre validation checks and processing of the config file to interpolate env vars returning a config dict ready to be tested against the schema. """ - config = interpolate_environment_variables(config) - return config + return interpolate_environment_variables(config) def load(config_details): - config, working_dir, filename = config_details + working_dir, configs = config_details - processed_config = pre_process_config(config) - validate_against_fields_schema(processed_config) - - service_dicts = [] - - for service_name, service_dict in list(processed_config.items()): - loader = ServiceLoader( - working_dir=working_dir, - filename=filename, - service_name=service_name, - service_dict=service_dict) + def build_service(filename, service_name, service_dict): + loader = ServiceLoader(working_dir, filename, service_name, service_dict) service_dict = loader.make_service_dict() validate_paths(service_dict) - service_dicts.append(service_dict) + return service_dict - return service_dicts + def load_file(filename, config): + processed_config = pre_process_config(config) + validate_against_fields_schema(processed_config) + return [ + build_service(filename, name, service_config) + for name, service_config in processed_config.items() + ] + + def merge_services(base, override): + return { + name: merge_service_dicts(base.get(name, {}), override.get(name, {})) + for name in set(base) | set(override) + } + + def combine_configs(override, base): + service_dicts = load_file(base.filename, base.config) + if not override: + return service_dicts + + return merge_service_dicts(base.config, override.config) + + return reduce(combine_configs, configs, None) class ServiceLoader(object): diff --git a/tests/unit/config_test.py b/tests/unit/config_test.py index ff80270e6..0347e443f 100644 --- a/tests/unit/config_test.py +++ b/tests/unit/config_test.py @@ -26,10 +26,16 @@ def service_sort(services): return sorted(services, key=itemgetter('name')) +def build_config_details(contents, working_dir, filename): + return config.ConfigDetails( + working_dir, + [config.ConfigFile(filename, contents)]) + + class ConfigTest(unittest.TestCase): def test_load(self): service_dicts = config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox'}, 'bar': {'image': 'busybox', 'environment': ['FOO=1']}, @@ -57,7 +63,7 @@ class ConfigTest(unittest.TestCase): def test_load_throws_error_when_not_dict(self): with self.assertRaises(ConfigurationError): config.load( - config.ConfigDetails( + build_config_details( {'web': 'busybox:latest'}, 'working_dir', 'filename.yml' @@ -68,7 +74,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaises(ConfigurationError): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: config.load( - config.ConfigDetails( + build_config_details( {invalid_name: {'image': 'busybox'}}, 'working_dir', 'filename.yml' @@ -79,7 +85,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service name: 1 needs to be a string, eg '1'" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {1: {'image': 'busybox'}}, 'working_dir', 'filename.yml' @@ -89,7 +95,7 @@ class ConfigTest(unittest.TestCase): def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( - config.ConfigDetails( + build_config_details( {valid_name: {'image': 'busybox'}}, 'tests/fixtures/extends', 'common.yml' @@ -101,7 +107,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]: config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': 'busybox', 'ports': invalid_ports}}, 'working_dir', 'filename.yml' @@ -112,7 +118,7 @@ class ConfigTest(unittest.TestCase): valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], [8000], ["49153-49154:3002-3003"]] for ports in valid_ports: config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': 'busybox', 'ports': ports}}, 'working_dir', 'filename.yml' @@ -123,7 +129,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "(did you mean 'privileged'?)" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'privilige': 'something'}, }, @@ -136,7 +142,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service 'foo' has both an image and build path specified." with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'build': '.'}, }, @@ -149,7 +155,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service 'foo' configuration key 'links' contains an invalid type, it should be an array" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'links': 'an_link'}, }, @@ -162,7 +168,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Top level object needs to be a dictionary." with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( ['foo', 'lol'], 'tests/fixtures/extends', 'filename.yml' @@ -173,7 +179,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "has non-unique elements" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'build': '.', 'devices': ['/dev/foo:/dev/foo', '/dev/foo:/dev/foo']} }, @@ -187,7 +193,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg += ", which is an invalid type, it should be a string" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'build': '.', 'command': [1]} }, @@ -200,7 +206,7 @@ class ConfigTest(unittest.TestCase): expected_error_msg = "Service 'web' has both an image and alternate Dockerfile." with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': 'busybox', 'dockerfile': 'Dockerfile.alt'}}, 'working_dir', 'filename.yml' @@ -212,7 +218,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'extra_hosts': 'somehost:162.242.195.82' @@ -227,7 +233,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'extra_hosts': [ @@ -244,7 +250,7 @@ class ConfigTest(unittest.TestCase): expose_values = [["8000"], [8000]] for expose in expose_values: service = config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'expose': expose @@ -259,7 +265,7 @@ class ConfigTest(unittest.TestCase): entrypoint_values = [["sh"], "sh"] for entrypoint in entrypoint_values: service = config.load( - config.ConfigDetails( + build_config_details( {'web': { 'image': 'busybox', 'entrypoint': entrypoint @@ -331,16 +337,16 @@ class InterpolationTest(unittest.TestCase): def test_unset_variable_produces_warning(self): os.environ.pop('FOO', None) os.environ.pop('BAR', None) - config_details = config.ConfigDetails( - config={ + config_details = build_config_details( + { 'web': { 'image': '${FOO}', 'command': '${BAR}', 'container_name': '${BAR}', }, }, - working_dir='.', - filename=None, + '.', + None, ) with mock.patch('compose.config.interpolation.log') as log: @@ -355,7 +361,7 @@ class InterpolationTest(unittest.TestCase): def test_invalid_interpolation(self): with self.assertRaises(config.ConfigurationError) as cm: config.load( - config.ConfigDetails( + build_config_details( {'web': {'image': '${'}}, 'working_dir', 'filename.yml' @@ -371,10 +377,10 @@ class InterpolationTest(unittest.TestCase): def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' d = config.load( - config.ConfigDetails( - config={'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, - working_dir='.', - filename=None, + build_config_details( + {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, + '.', + None, ) )[0] self.assertEqual(d['volumes'], ['/host/path:/container/path']) @@ -649,7 +655,7 @@ class MemoryOptionsTest(unittest.TestCase): ) with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'image': 'busybox', 'memswap_limit': 2000000}, }, @@ -660,7 +666,7 @@ class MemoryOptionsTest(unittest.TestCase): def test_validation_with_correct_memswap_values(self): service_dict = config.load( - config.ConfigDetails( + build_config_details( {'foo': {'image': 'busybox', 'mem_limit': 1000000, 'memswap_limit': 2000000}}, 'tests/fixtures/extends', 'common.yml' @@ -670,7 +676,7 @@ class MemoryOptionsTest(unittest.TestCase): def test_memswap_can_be_a_string(self): service_dict = config.load( - config.ConfigDetails( + build_config_details( {'foo': {'image': 'busybox', 'mem_limit': "1G", 'memswap_limit': "512M"}}, 'tests/fixtures/extends', 'common.yml' @@ -780,26 +786,26 @@ class EnvTest(unittest.TestCase): os.environ['CONTAINERENV'] = '/host/tmp' service_dict = config.load( - config.ConfigDetails( - config={'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, - working_dir="tests/fixtures/env", - filename=None, + build_config_details( + {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, + "tests/fixtures/env", + None, ) )[0] self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp'])) service_dict = config.load( - config.ConfigDetails( - config={'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, - working_dir="tests/fixtures/env", - filename=None, + build_config_details( + {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, + "tests/fixtures/env", + None, ) )[0] self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp'])) def load_from_filename(filename): - return config.load(config.find('.', filename)) + return config.load(config.find('.', [filename])) class ExtendsTest(unittest.TestCase): @@ -885,7 +891,7 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation_empty_dictionary(self): with self.assertRaisesRegexp(ConfigurationError, 'service'): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'image': 'busybox', 'extends': {}}, }, @@ -897,7 +903,7 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation_missing_service_key(self): with self.assertRaisesRegexp(ConfigurationError, "'service' is a required property"): config.load( - config.ConfigDetails( + build_config_details( { 'web': {'image': 'busybox', 'extends': {'file': 'common.yml'}}, }, @@ -910,7 +916,7 @@ class ExtendsTest(unittest.TestCase): expected_error_msg = "Unsupported config option for 'web' service: 'rogue_key'" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': { 'image': 'busybox', @@ -930,7 +936,7 @@ class ExtendsTest(unittest.TestCase): expected_error_msg = "Service 'web' configuration key 'extends' 'file' contains an invalid type" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( - config.ConfigDetails( + build_config_details( { 'web': { 'image': 'busybox', @@ -955,7 +961,7 @@ class ExtendsTest(unittest.TestCase): def test_extends_validation_valid_config(self): service = config.load( - config.ConfigDetails( + build_config_details( { 'web': {'image': 'busybox', 'extends': {'service': 'web', 'file': 'common.yml'}}, }, @@ -1093,7 +1099,7 @@ class BuildPathTest(unittest.TestCase): def test_nonexistent_path(self): with self.assertRaises(ConfigurationError): config.load( - config.ConfigDetails( + build_config_details( { 'foo': {'build': 'nonexistent.path'}, },