from __future__ import absolute_import from __future__ import unicode_literals import codecs import functools import logging import operator import os import string import sys from collections import namedtuple import six import yaml from cached_property import cached_property from ..const import COMPOSEFILE_V1 as V1 from ..const import COMPOSEFILE_V2_0 as V2_0 from ..utils import build_string_dict from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError from .errors import VERSION_EXPLANATION from .interpolation import interpolate_environment_variables from .sort_services import get_container_name_from_network_mode from .sort_services import get_service_name_from_network_mode from .sort_services import sort_service_dicts from .types import parse_extra_hosts from .types import parse_restart_spec from .types import ServiceLink from .types import VolumeFromSpec from .types import VolumeSpec from .validation import match_named_volumes from .validation import validate_against_config_schema from .validation import validate_config_section from .validation import validate_depends_on from .validation import validate_extends_file_path from .validation import validate_network_mode from .validation import validate_service_constraints from .validation import validate_top_level_object from .validation import validate_ulimits DOCKER_CONFIG_KEYS = [ 'cap_add', 'cap_drop', 'cgroup_parent', 'command', 'cpu_quota', 'cpu_shares', 'cpuset', 'detach', 'devices', 'dns', 'dns_search', 'domainname', 'entrypoint', 'env_file', 'environment', 'extra_hosts', 'hostname', 'image', 'ipc', 'labels', 'links', 'mac_address', 'mem_limit', 'memswap_limit', 'net', 'pid', 'ports', 'privileged', 'read_only', 'restart', 'security_opt', 'shm_size', 'stdin_open', 'stop_signal', 'tty', 'user', 'volume_driver', 'volumes', 'volumes_from', 'working_dir', ] ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'build', 'container_name', 'dockerfile', 'log_driver', 'log_opt', 'logging', 'network_mode', ] DOCKER_VALID_URL_PREFIXES = ( 'http://', 'https://', 'git://', 'github.com/', 'git@', ) SUPPORTED_FILENAMES = [ 'docker-compose.yml', 'docker-compose.yaml', ] DEFAULT_OVERRIDE_FILENAME = 'docker-compose.override.yml' log = logging.getLogger(__name__) class ConfigDetails(namedtuple('_ConfigDetails', 'working_dir config_files')): """ :param working_dir: the directory to use for relative paths in the config :type working_dir: string :param config_files: list of configuration files to load :type config_files: list of :class:`ConfigFile` """ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): """ :param filename: filename of the config file :type filename: string :param config: contents of the config file :type config: :class:`dict` """ @classmethod def from_filename(cls, filename): return cls(filename, load_yaml(filename)) @cached_property def version(self): if 'version' not in self.config: return V1 version = self.config['version'] if isinstance(version, dict): log.warn('Unexpected type for "version" key in "{}". Assuming ' '"version" is the name of a service, and defaulting to ' 'Compose file version 1.'.format(self.filename)) return V1 if not isinstance(version, six.string_types): raise ConfigurationError( 'Version in "{}" is invalid - it should be a string.' .format(self.filename)) if version == '1': raise ConfigurationError( 'Version in "{}" is invalid. {}' .format(self.filename, VERSION_EXPLANATION)) if version == '2': version = V2_0 if version != V2_0: raise ConfigurationError( 'Version in "{}" is unsupported. {}' .format(self.filename, VERSION_EXPLANATION)) return version def get_service(self, name): return self.get_service_dicts()[name] def get_service_dicts(self): return self.config if self.version == V1 else self.config.get('services', {}) def get_volumes(self): return {} if self.version == V1 else self.config.get('volumes', {}) def get_networks(self): return {} if self.version == V1 else self.config.get('networks', {}) class Config(namedtuple('_Config', 'version services volumes networks')): """ :param version: configuration version :type version: int :param services: List of service description dictionaries :type services: :class:`list` :param volumes: Dictionary mapping volume names to description dictionaries :type volumes: :class:`dict` :param networks: Dictionary mapping network names to description dictionaries :type networks: :class:`dict` """ class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name config')): @classmethod def with_abs_paths(cls, working_dir, filename, name, config): if not working_dir: raise ValueError("No working_dir for ServiceConfig.") return cls( os.path.abspath(working_dir), os.path.abspath(filename) if filename else filename, name, config) def find(base_dir, filenames): if filenames == ['-']: return ConfigDetails( os.getcwd(), [ConfigFile(None, yaml.safe_load(sys.stdin))]) if filenames: filenames = [os.path.join(base_dir, f) for f in filenames] else: filenames = get_default_config_files(base_dir) log.debug("Using configuration files: {}".format(",".join(filenames))) return ConfigDetails( os.path.dirname(filenames[0]), [ConfigFile.from_filename(f) for f in filenames]) def validate_config_version(config_files): main_file = config_files[0] validate_top_level_object(main_file) for next_file in config_files[1:]: validate_top_level_object(next_file) if main_file.version != next_file.version: raise ConfigurationError( "Version mismatch: file {0} specifies version {1} but " "extension file {2} uses version {3}".format( main_file.filename, main_file.version, next_file.filename, next_file.version)) def get_default_config_files(base_dir): (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir) if not candidates: raise ComposeFileNotFound(SUPPORTED_FILENAMES) winner = candidates[0] if len(candidates) > 1: log.warn("Found multiple config files with supported names: %s", ", ".join(candidates)) log.warn("Using %s\n", winner) return [os.path.join(path, winner)] + get_default_override_file(path) def get_default_override_file(path): override_filename = os.path.join(path, DEFAULT_OVERRIDE_FILENAME) return [override_filename] if os.path.exists(override_filename) else [] def find_candidates_in_parent_dirs(filenames, path): """ Given a directory path to start, looks for filenames in the directory, and then each parent directory successively, until found. Returns tuple (candidates, path). """ candidates = [filename for filename in filenames if os.path.exists(os.path.join(path, filename))] if not candidates: parent_dir = os.path.join(path, '..') if os.path.abspath(parent_dir) != os.path.abspath(path): return find_candidates_in_parent_dirs(filenames, parent_dir) return (candidates, path) def load(config_details): """Load the configuration from a working directory and a list of configuration files. Files are loaded in order, and merged on top of each other to create the final configuration. Return a fully interpolated, extended and validated configuration. """ validate_config_version(config_details.config_files) processed_files = [ process_config_file(config_file) for config_file in config_details.config_files ] config_details = config_details._replace(config_files=processed_files) main_file = config_details.config_files[0] volumes = load_mapping( config_details.config_files, 'get_volumes', 'Volume' ) networks = load_mapping( config_details.config_files, 'get_networks', 'Network' ) service_dicts = load_services( config_details.working_dir, main_file, [file.get_service_dicts() for file in config_details.config_files]) if main_file.version != V1: for service_dict in service_dicts: match_named_volumes(service_dict, volumes) return Config(main_file.version, service_dicts, volumes, networks) def load_mapping(config_files, get_func, entity_type): mapping = {} for config_file in config_files: for name, config in getattr(config_file, get_func)().items(): mapping[name] = config or {} if not config: continue external = config.get('external') if external: if len(config.keys()) > 1: raise ConfigurationError( '{} {} declared as external but specifies' ' additional attributes ({}). '.format( entity_type, name, ', '.join([k for k in config.keys() if k != 'external']) ) ) if isinstance(external, dict): config['external_name'] = external.get('name') else: config['external_name'] = name mapping[name] = config if 'driver_opts' in config: config['driver_opts'] = build_string_dict( config['driver_opts'] ) return mapping def load_services(working_dir, config_file, service_configs): def build_service(service_name, service_dict, service_names): service_config = ServiceConfig.with_abs_paths( working_dir, config_file.filename, service_name, service_dict) resolver = ServiceExtendsResolver(service_config, config_file) service_dict = process_service(resolver.run()) service_config = service_config._replace(config=service_dict) validate_service(service_config, service_names, config_file.version) service_dict = finalize_service( service_config, service_names, config_file.version) return service_dict def build_services(service_config): service_names = service_config.keys() return sort_service_dicts([ build_service(name, service_dict, service_names) for name, service_dict in service_config.items() ]) def merge_services(base, override): all_service_names = set(base) | set(override) return { name: merge_service_dicts_from_files( base.get(name, {}), override.get(name, {}), config_file.version) for name in all_service_names } service_config = service_configs[0] for next_config in service_configs[1:]: service_config = merge_services(service_config, next_config) return build_services(service_config) def interpolate_config_section(filename, config, section): validate_config_section(filename, config, section) return interpolate_environment_variables(config, section) def process_config_file(config_file, service_name=None): services = interpolate_config_section( config_file.filename, config_file.get_service_dicts(), 'service') if config_file.version == V2_0: processed_config = dict(config_file.config) processed_config['services'] = services processed_config['volumes'] = interpolate_config_section( config_file.filename, config_file.get_volumes(), 'volume') processed_config['networks'] = interpolate_config_section( config_file.filename, config_file.get_networks(), 'network') if config_file.version == V1: processed_config = services config_file = config_file._replace(config=processed_config) validate_against_config_schema(config_file) if service_name and service_name not in services: raise ConfigurationError( "Cannot extend service '{}' in {}: Service not found".format( service_name, config_file.filename)) return config_file class ServiceExtendsResolver(object): def __init__(self, service_config, config_file, already_seen=None): self.service_config = service_config self.working_dir = service_config.working_dir self.already_seen = already_seen or [] self.config_file = config_file @property def signature(self): return self.service_config.filename, self.service_config.name def detect_cycle(self): if self.signature in self.already_seen: raise CircularReference(self.already_seen + [self.signature]) def run(self): self.detect_cycle() if 'extends' in self.service_config.config: service_dict = self.resolve_extends(*self.validate_and_construct_extends()) return self.service_config._replace(config=service_dict) return self.service_config def validate_and_construct_extends(self): extends = self.service_config.config['extends'] if not isinstance(extends, dict): extends = {'service': extends} config_path = self.get_extended_config_path(extends) service_name = extends['service'] extends_file = ConfigFile.from_filename(config_path) validate_config_version([self.config_file, extends_file]) extended_file = process_config_file( extends_file, service_name=service_name) service_config = extended_file.get_service(service_name) return config_path, service_config, service_name def resolve_extends(self, extended_config_path, service_dict, service_name): resolver = ServiceExtendsResolver( ServiceConfig.with_abs_paths( os.path.dirname(extended_config_path), extended_config_path, service_name, service_dict), self.config_file, already_seen=self.already_seen + [self.signature]) service_config = resolver.run() other_service_dict = process_service(service_config) validate_extended_service_dict( other_service_dict, extended_config_path, service_name) return merge_service_dicts( other_service_dict, self.service_config.config, self.config_file.version) def get_extended_config_path(self, extends_options): """Service we are extending either has a value for 'file' set, which we need to obtain a full path too or we are extending from a service defined in our own file. """ filename = self.service_config.filename validate_extends_file_path( self.service_config.name, extends_options, filename) if 'file' in extends_options: return expand_path(self.working_dir, extends_options['file']) return filename def resolve_environment(service_dict): """Unpack any environment variables from an env_file, if set. Interpolate environment values if set. """ env = {} for env_file in service_dict.get('env_file', []): env.update(env_vars_from_file(env_file)) env.update(parse_environment(service_dict.get('environment'))) return dict(resolve_env_var(k, v) for k, v in six.iteritems(env)) def resolve_build_args(build): args = parse_build_arguments(build.get('args')) return dict(resolve_env_var(k, v) for k, v in six.iteritems(args)) def validate_extended_service_dict(service_dict, filename, service): error_prefix = "Cannot extend service '%s' in %s:" % (service, filename) if 'links' in service_dict: raise ConfigurationError( "%s services with 'links' cannot be extended" % error_prefix) if 'volumes_from' in service_dict: raise ConfigurationError( "%s services with 'volumes_from' cannot be extended" % error_prefix) if 'net' in service_dict: if get_container_name_from_network_mode(service_dict['net']): raise ConfigurationError( "%s services with 'net: container' cannot be extended" % error_prefix) if 'network_mode' in service_dict: if get_service_name_from_network_mode(service_dict['network_mode']): raise ConfigurationError( "%s services with 'network_mode: service' cannot be extended" % error_prefix) if 'depends_on' in service_dict: raise ConfigurationError( "%s services with 'depends_on' cannot be extended" % error_prefix) def validate_service(service_config, service_names, version): service_dict, service_name = service_config.config, service_config.name validate_service_constraints(service_dict, service_name, version) validate_paths(service_dict) validate_ulimits(service_config) validate_network_mode(service_config, service_names) validate_depends_on(service_config, service_names) if not service_dict.get('image') and has_uppercase(service_name): raise ConfigurationError( "Service '{name}' contains uppercase characters which are not valid " "as part of an image name. Either use a lowercase service name or " "use the `image` field to set a custom name for the service image." .format(name=service_name)) def process_service(service_config): working_dir = service_config.working_dir service_dict = dict(service_config.config) if 'env_file' in service_dict: service_dict['env_file'] = [ expand_path(working_dir, path) for path in to_list(service_dict['env_file']) ] if 'build' in service_dict: if isinstance(service_dict['build'], six.string_types): service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) elif isinstance(service_dict['build'], dict) and 'context' in service_dict['build']: path = service_dict['build']['context'] service_dict['build']['context'] = resolve_build_path(working_dir, path) if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) if 'labels' in service_dict: service_dict['labels'] = parse_labels(service_dict['labels']) if 'extra_hosts' in service_dict: service_dict['extra_hosts'] = parse_extra_hosts(service_dict['extra_hosts']) for field in ['dns', 'dns_search']: if field in service_dict: service_dict[field] = to_list(service_dict[field]) return service_dict def finalize_service(service_config, service_names, version): service_dict = dict(service_config.config) if 'environment' in service_dict or 'env_file' in service_dict: service_dict['environment'] = resolve_environment(service_dict) service_dict.pop('env_file', None) if 'volumes_from' in service_dict: service_dict['volumes_from'] = [ VolumeFromSpec.parse(vf, service_names, version) for vf in service_dict['volumes_from'] ] if 'volumes' in service_dict: service_dict['volumes'] = [ VolumeSpec.parse(v) for v in service_dict['volumes']] if 'net' in service_dict: network_mode = service_dict.pop('net') container_name = get_container_name_from_network_mode(network_mode) if container_name and container_name in service_names: service_dict['network_mode'] = 'service:{}'.format(container_name) else: service_dict['network_mode'] = network_mode if 'networks' in service_dict: service_dict['networks'] = parse_networks(service_dict['networks']) if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) normalize_build(service_dict, service_config.working_dir) service_dict['name'] = service_config.name return normalize_v1_service_format(service_dict) def normalize_v1_service_format(service_dict): if 'log_driver' in service_dict or 'log_opt' in service_dict: if 'logging' not in service_dict: service_dict['logging'] = {} if 'log_driver' in service_dict: service_dict['logging']['driver'] = service_dict['log_driver'] del service_dict['log_driver'] if 'log_opt' in service_dict: service_dict['logging']['options'] = service_dict['log_opt'] del service_dict['log_opt'] if 'dockerfile' in service_dict: service_dict['build'] = service_dict.get('build', {}) service_dict['build'].update({ 'dockerfile': service_dict.pop('dockerfile') }) return service_dict def merge_service_dicts_from_files(base, override, version): """When merging services from multiple files we need to merge the `extends` field. This is not handled by `merge_service_dicts()` which is used to perform the `extends`. """ new_service = merge_service_dicts(base, override, version) if 'extends' in override: new_service['extends'] = override['extends'] elif 'extends' in base: new_service['extends'] = base['extends'] return new_service class MergeDict(dict): """A dict-like object responsible for merging two dicts into one.""" def __init__(self, base, override): self.base = base self.override = override def needs_merge(self, field): return field in self.base or field in self.override def merge_field(self, field, merge_func, default=None): if not self.needs_merge(field): return self[field] = merge_func( self.base.get(field, default), self.override.get(field, default)) def merge_mapping(self, field, parse_func): if not self.needs_merge(field): return self[field] = parse_func(self.base.get(field)) self[field].update(parse_func(self.override.get(field))) def merge_sequence(self, field, parse_func): def parse_sequence_func(seq): return to_mapping((parse_func(item) for item in seq), 'merge_field') if not self.needs_merge(field): return merged = parse_sequence_func(self.base.get(field, [])) merged.update(parse_sequence_func(self.override.get(field, []))) self[field] = [item.repr() for item in merged.values()] def merge_scalar(self, field): if self.needs_merge(field): self[field] = self.override.get(field, self.base.get(field)) def merge_service_dicts(base, override, version): md = MergeDict(base, override) md.merge_mapping('environment', parse_environment) md.merge_mapping('labels', parse_labels) md.merge_mapping('ulimits', parse_ulimits) md.merge_mapping('networks', parse_networks) md.merge_sequence('links', ServiceLink.parse) for field in ['volumes', 'devices']: md.merge_field(field, merge_path_mappings) for field in [ 'depends_on', 'expose', 'external_links', 'ports', 'volumes_from', ]: md.merge_field(field, operator.add, default=[]) for field in ['dns', 'dns_search', 'env_file']: md.merge_field(field, merge_list_or_string) for field in set(ALLOWED_KEYS) - set(md): md.merge_scalar(field) if version == V1: legacy_v1_merge_image_or_build(md, base, override) elif md.needs_merge('build'): md['build'] = merge_build(md, base, override) return dict(md) def merge_build(output, base, override): def to_dict(service): build_config = service.get('build', {}) if isinstance(build_config, six.string_types): return {'context': build_config} return build_config md = MergeDict(to_dict(base), to_dict(override)) md.merge_scalar('context') md.merge_scalar('dockerfile') md.merge_mapping('args', parse_build_arguments) return dict(md) def legacy_v1_merge_image_or_build(output, base, override): output.pop('image', None) output.pop('build', None) if 'image' in override: output['image'] = override['image'] elif 'build' in override: output['build'] = override['build'] elif 'image' in base: output['image'] = base['image'] elif 'build' in base: output['build'] = base['build'] def merge_environment(base, override): env = parse_environment(base) env.update(parse_environment(override)) return env def split_env(env): if isinstance(env, six.binary_type): env = env.decode('utf-8', 'replace') if '=' in env: return env.split('=', 1) else: return env, None def split_label(label): if '=' in label: return label.split('=', 1) else: return label, '' def parse_dict_or_list(split_func, type_name, arguments): if not arguments: return {} if isinstance(arguments, list): return dict(split_func(e) for e in arguments) if isinstance(arguments, dict): return dict(arguments) raise ConfigurationError( "%s \"%s\" must be a list or mapping," % (type_name, arguments) ) parse_build_arguments = functools.partial(parse_dict_or_list, split_env, 'build arguments') parse_environment = functools.partial(parse_dict_or_list, split_env, 'environment') parse_labels = functools.partial(parse_dict_or_list, split_label, 'labels') parse_networks = functools.partial(parse_dict_or_list, lambda k: (k, None), 'networks') def parse_ulimits(ulimits): if not ulimits: return {} if isinstance(ulimits, dict): return dict(ulimits) def resolve_env_var(key, val): if val is not None: return key, val elif key in os.environ: return key, os.environ[key] else: return key, None def env_vars_from_file(filename): """ Read in a line delimited file of environment variables. """ if not os.path.exists(filename): raise ConfigurationError("Couldn't find env file: %s" % filename) env = {} for line in codecs.open(filename, 'r', 'utf-8'): line = line.strip() if line and not line.startswith('#'): k, v = split_env(line) env[k] = v return env def resolve_volume_paths(working_dir, service_dict): return [ resolve_volume_path(working_dir, volume) for volume in service_dict['volumes'] ] def resolve_volume_path(working_dir, volume): container_path, host_path = split_path_mapping(volume) if host_path is not None: if host_path.startswith('.'): host_path = expand_path(working_dir, host_path) host_path = os.path.expanduser(host_path) return u"{}:{}".format(host_path, container_path) else: return container_path def normalize_build(service_dict, working_dir): if 'build' in service_dict: build = {} # Shortcut where specifying a string is treated as the build context if isinstance(service_dict['build'], six.string_types): build['context'] = service_dict.pop('build') else: build.update(service_dict['build']) if 'args' in build: build['args'] = build_string_dict(resolve_build_args(build)) service_dict['build'] = build def resolve_build_path(working_dir, build_path): if is_url(build_path): return build_path return expand_path(working_dir, build_path) def is_url(build_path): return build_path.startswith(DOCKER_VALID_URL_PREFIXES) def validate_paths(service_dict): if 'build' in service_dict: build = service_dict.get('build', {}) if isinstance(build, six.string_types): build_path = build elif isinstance(build, dict) and 'context' in build: build_path = build['context'] else: # We have a build section but no context, so nothing to validate return if ( not is_url(build_path) and (not os.path.exists(build_path) or not os.access(build_path, os.R_OK)) ): raise ConfigurationError( "build path %s either does not exist, is not accessible, " "or is not a valid URL." % build_path) def merge_path_mappings(base, override): d = dict_from_path_mappings(base) d.update(dict_from_path_mappings(override)) return path_mappings_from_dict(d) def dict_from_path_mappings(path_mappings): if path_mappings: return dict(split_path_mapping(v) for v in path_mappings) else: return {} def path_mappings_from_dict(d): return [join_path_mapping(v) for v in d.items()] def split_path_mapping(volume_path): """ Ascertain if the volume_path contains a host path as well as a container path. Using splitdrive so windows absolute paths won't cause issues with splitting on ':'. """ # splitdrive has limitations when it comes to relative paths, so when it's # relative, handle special case to set the drive to '' if volume_path.startswith('.') or volume_path.startswith('~'): drive, volume_config = '', volume_path else: drive, volume_config = os.path.splitdrive(volume_path) if ':' in volume_config: (host, container) = volume_config.split(':', 1) return (container, drive + host) else: return (volume_path, None) def join_path_mapping(pair): (container, host) = pair if host is None: return container else: return ":".join((host, container)) def expand_path(working_dir, path): return os.path.abspath(os.path.join(working_dir, os.path.expanduser(path))) def merge_list_or_string(base, override): return to_list(base) + to_list(override) def to_list(value): if value is None: return [] elif isinstance(value, six.string_types): return [value] else: return value def to_mapping(sequence, key_field): return {getattr(item, key_field): item for item in sequence} def has_uppercase(name): return any(char in string.ascii_uppercase for char in name) def load_yaml(filename): try: with open(filename, 'r') as fh: return yaml.safe_load(fh) except (IOError, yaml.YAMLError) as e: error_name = getattr(e, '__module__', '') + '.' + e.__class__.__name__ raise ConfigurationError(u"{}: {}".format(error_name, e))