Merge pull request #236 from rail44/feature-support-volumes-from

Support volumes_from option

fig/project.py

@@ -2,6 +2,8 @@ from __future__ import unicode_literals
 from __future__ import absolute_import
 import logging
 from .service import Service
+from .container import Container
+from .packages.docker.errors import APIError
 
 log = logging.getLogger(__name__)
 
@@ -18,11 +20,13 @@ def sort_service_dicts(services):
         if n['name'] in temporary_marked:
             if n['name'] in get_service_names(n.get('links', [])):
                 raise DependencyError('A service can not link to itself: %s' % n['name'])
+            if n['name'] in n.get('volumes_from', []):
+                raise DependencyError('A service can not mount itself as volume: %s' % n['name'])
             else:
                 raise DependencyError('Circular import between %s' % ' and '.join(temporary_marked))
         if n in unmarked:
             temporary_marked.add(n['name'])
-            dependents = [m for m in services if n['name'] in get_service_names(m.get('links', []))]
+            dependents = [m for m in services if (n['name'] in get_service_names(m.get('links', []))) or (n['name'] in m.get('volumes_from', []))]
             for m in dependents:
                 visit(m)
             temporary_marked.remove(n['name'])
@@ -50,22 +54,10 @@ class Project(object):
         """
         project = cls(name, [], client)
         for service_dict in sort_service_dicts(service_dicts):
-            # Reference links by object
+            links = project.get_links(service_dict)
-            links = []
+            volumes_from = project.get_volumes_from(service_dict)
-            if 'links' in service_dict:
-                for link in service_dict.get('links', []):
-                    if ':' in link:
-                        service_name, link_name = link.split(':', 1)
-                    else:
-                        service_name, link_name = link, None
-                    try:
-                        links.append((project.get_service(service_name), link_name))
-                    except NoSuchService:
-                        raise ConfigurationError('Service "%s" has a link to service "%s" which does not exist.' % (service_dict['name'], service_name))
 
-                del service_dict['links']
+            project.services.append(Service(client=client, project=name, links=links, volumes_from=volumes_from, **service_dict))
-
-            project.services.append(Service(client=client, project=name, links=links, **service_dict))
         return project
 
     @classmethod
@@ -119,6 +111,37 @@ class Project(object):
             [uniques.append(s) for s in services if s not in uniques]
             return uniques
 
+    def get_links(self, service_dict):
+        links = []
+        if 'links' in service_dict:
+            for link in service_dict.get('links', []):
+                if ':' in link:
+                    service_name, link_name = link.split(':', 1)
+                else:
+                    service_name, link_name = link, None
+                try:
+                    links.append((self.get_service(service_name), link_name))
+                except NoSuchService:
+                    raise ConfigurationError('Service "%s" has a link to service "%s" which does not exist.' % (service_dict['name'], service_name))
+            del service_dict['links']
+        return links
+
+    def get_volumes_from(self, service_dict):
+        volumes_from = []
+        if 'volumes_from' in service_dict:
+            for volume_name in service_dict.get('volumes_from', []):
+                try:
+                    service = self.get_service(volume_name)
+                    volumes_from.append(service)
+                except NoSuchService:
+                    try:
+                        container = Container.from_id(client, volume_name)
+                        volumes_from.append(Container.from_id(client, volume_name))
+                    except APIError:
+                        raise ConfigurationError('Service "%s" mounts volumes from "%s", which is not the name of a service or container.' % (service_dict['name'], volume_name))
+            del service_dict['volumes_from']
+        return volumes_from
+
     def start(self, service_names=None, **options):
         for service in self.get_services(service_names):
             service.start(**options)

fig/service.py

@@ -11,7 +11,7 @@ from .progress_stream import stream_output, StreamOutputError
 log = logging.getLogger(__name__)
 
 
-DOCKER_CONFIG_KEYS = ['image', 'command', 'hostname', 'user', 'detach', 'stdin_open', 'tty', 'mem_limit', 'ports', 'environment', 'dns', 'volumes', 'volumes_from', 'entrypoint', 'privileged', 'net']
+DOCKER_CONFIG_KEYS = ['image', 'command', 'hostname', 'user', 'detach', 'stdin_open', 'tty', 'mem_limit', 'ports', 'environment', 'dns', 'volumes', 'entrypoint', 'privileged', 'volumes_from', 'net']
 DOCKER_CONFIG_HINTS = {
     'link'      : 'links',
     'port'      : 'ports',
@@ -39,7 +39,7 @@ class ConfigError(ValueError):
 
 
 class Service(object):
-    def __init__(self, name, client=None, project='default', links=[], **options):
+    def __init__(self, name, client=None, project='default', links=[], volumes_from=[], **options):
         if not re.match('^%s+$' % VALID_NAME_CHARS, name):
             raise ConfigError('Invalid service name "%s" - only %s are allowed' % (name, VALID_NAME_CHARS))
         if not re.match('^%s+$' % VALID_NAME_CHARS, project):
@@ -60,6 +60,7 @@ class Service(object):
         self.client = client
         self.project = project
         self.links = links or []
+        self.volumes_from = volumes_from or []
         self.options = options
 
     def containers(self, stopped=False, one_off=False):
@@ -190,7 +191,7 @@ class Service(object):
 
         options = dict(override_options)
         new_container = self.create_container(**options)
-        self.start_container(new_container, volumes_from=intermediate_container.id)
+        self.start_container(new_container, intermediate_container=intermediate_container)
 
         intermediate_container.remove()
 
@@ -203,7 +204,7 @@ class Service(object):
             log.info("Starting %s..." % container.name)
             return self.start_container(container, **options)
 
-    def start_container(self, container=None, volumes_from=None, **override_options):
+    def start_container(self, container=None, intermediate_container=None,**override_options):
         if container is None:
             container = self.create_container(**override_options)
 
@@ -235,7 +236,7 @@ class Service(object):
             links=self._get_links(link_to_self=override_options.get('one_off', False)),
             port_bindings=port_bindings,
             binds=volume_bindings,
-            volumes_from=volumes_from,
+            volumes_from=self._get_volumes_from(intermediate_container),
             privileged=privileged,
             network_mode=net,
         )
@@ -282,6 +283,20 @@ class Service(object):
                 links.append((container.name, container.name_without_project))
         return links
 
+    def _get_volumes_from(self, intermediate_container=None):
+        volumes_from = []
+        for v in self.volumes_from:
+            if isinstance(v, Service):
+                for container in v.containers(stopped=True):
+                    volumes_from.append(container.id)
+            elif isinstance(v, Container):
+                volumes_from.append(v.id)
+
+        if intermediate_container:
+            volumes_from.append(intermediate_container.id)
+
+        return volumes_from
+
     def _get_container_create_options(self, override_options, one_off=False):
         container_options = dict((k, self.options[k]) for k in DOCKER_CONFIG_KEYS if k in self.options)
         container_options.update(override_options)

tests/integration/service_test.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 from __future__ import absolute_import
 from fig import Service
 from fig.service import CannotBeScaledError
+from fig.container import Container
 from fig.packages.docker.errors import APIError
 from .testcases import DockerClientTestCase
 
@@ -96,6 +97,16 @@ class ServiceTest(DockerClientTestCase):
         service.start_container(container)
         self.assertIn('/host-tmp', container.inspect()['Volumes'])
 
+    def test_create_container_with_volumes_from(self):
+        volume_service = self.create_service('data')
+        volume_container_1 = volume_service.create_container()
+        volume_container_2 = Container.create(self.client, image='busybox:latest', command=["/bin/sleep", "300"])
+        host_service = self.create_service('host', volumes_from=[volume_service, volume_container_2])
+        host_container = host_service.create_container()
+        host_service.start_container(host_container)
+        self.assertIn(volume_container_1.id, host_container.inspect()['HostConfig']['VolumesFrom'])
+        self.assertIn(volume_container_2.id, host_container.inspect()['HostConfig']['VolumesFrom'])
+
     def test_recreate_containers(self):
         service = self.create_service(
             'db',
@@ -127,6 +138,7 @@ class ServiceTest(DockerClientTestCase):
         self.assertIn('FOO=2', new_container.dictionary['Config']['Env'])
         self.assertEqual(new_container.name, 'figtest_db_1')
         self.assertEqual(new_container.inspect()['Volumes']['/var/db'], volume_path)
+        self.assertIn(intermediate_container.id, new_container.dictionary['HostConfig']['VolumesFrom'])
 
         self.assertEqual(len(self.client.containers(all=True)), num_containers_before)
         self.assertNotEqual(old_container.id, new_container.id)

tests/unit/project_test.py

@@ -30,12 +30,19 @@ class ProjectTest(unittest.TestCase):
             },
             {
                 'name': 'db',
-                'image': 'busybox:latest'
+                'image': 'busybox:latest',
+                'volumes_from': ['volume']
+            },
+            {
+                'name': 'volume',
+                'image': 'busybox:latest',
+                'volumes': ['/tmp'],
             }
         ], None)
 
-        self.assertEqual(project.services[0].name, 'db')
+        self.assertEqual(project.services[0].name, 'volume')
-        self.assertEqual(project.services[1].name, 'web')
+        self.assertEqual(project.services[1].name, 'db')
+        self.assertEqual(project.services[2].name, 'web')
 
     def test_from_config(self):
         project = Project.from_config('figtest', {