Merge pull request #3178 from aanand/2774-off-one-offs

Remove one-off containers in `rm` and `down`

compose/cli/main.py

@@ -22,6 +22,7 @@ from ..const import DEFAULT_TIMEOUT
 from ..const import IS_WINDOWS_PLATFORM
 from ..progress_stream import StreamOutputError
 from ..project import NoSuchService
+from ..project import OneOffFilter
 from ..service import BuildAction
 from ..service import BuildError
 from ..service import ConvergenceStrategy
@@ -437,7 +438,7 @@ class TopLevelCommand(object):
         """
         containers = sorted(
             self.project.containers(service_names=options['SERVICE'], stopped=True) +
-            self.project.containers(service_names=options['SERVICE'], one_off=True),
+            self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only),
             key=attrgetter('name'))
 
         if options['-q']:
@@ -491,8 +492,21 @@ class TopLevelCommand(object):
         Options:
             -f, --force   Don't ask to confirm removal
             -v            Remove volumes associated with containers
+            -a, --all     Also remove one-off containers created by
+                          docker-compose run
         """
-        all_containers = self.project.containers(service_names=options['SERVICE'], stopped=True)
+        if options.get('--all'):
+            one_off = OneOffFilter.include
+        else:
+            log.warn(
+                'Not including one-off containers created by `docker-compose run`.\n'
+                'To include them, use `docker-compose rm --all`.\n'
+                'This will be the default behavior in the next version of Compose.\n')
+            one_off = OneOffFilter.exclude
+
+        all_containers = self.project.containers(
+            service_names=options['SERVICE'], stopped=True, one_off=one_off
+        )
         stopped_containers = [c for c in all_containers if not c.is_running]
 
         if len(stopped_containers) > 0:
@@ -501,7 +515,8 @@ class TopLevelCommand(object):
                     or yesno("Are you sure? [yN] ", default=False):
                 self.project.remove_stopped(
                     service_names=options['SERVICE'],
-                    v=options.get('-v', False)
+                    v=options.get('-v', False),
+                    one_off=one_off
                 )
         else:
             print("No stopped containers")

compose/project.py

@@ -6,6 +6,7 @@ import logging
 import operator
 from functools import reduce
 
+import enum
 from docker.errors import APIError
 
 from . import parallel
@@ -35,6 +36,24 @@ from .volume import ProjectVolumes
 log = logging.getLogger(__name__)
 
 
+@enum.unique
+class OneOffFilter(enum.Enum):
+    include = 0
+    exclude = 1
+    only = 2
+
+    @classmethod
+    def update_labels(cls, value, labels):
+        if value == cls.only:
+            labels.append('{0}={1}'.format(LABEL_ONE_OFF, "True"))
+        elif value == cls.exclude:
+            labels.append('{0}={1}'.format(LABEL_ONE_OFF, "False"))
+        elif value == cls.include:
+            pass
+        else:
+            raise ValueError("Invalid value for one_off: {}".format(repr(value)))
+
+
 class Project(object):
     """
     A collection of services.
@@ -46,11 +65,11 @@ class Project(object):
         self.volumes = volumes or ProjectVolumes({})
         self.networks = networks or ProjectNetworks({}, False)
 
-    def labels(self, one_off=False):
+    def labels(self, one_off=OneOffFilter.exclude):
-        return [
+        labels = ['{0}={1}'.format(LABEL_PROJECT, self.name)]
-            '{0}={1}'.format(LABEL_PROJECT, self.name),
+
-            '{0}={1}'.format(LABEL_ONE_OFF, "True" if one_off else "False"),
+        OneOffFilter.update_labels(one_off, labels)
-        ]
+        return labels
 
     @classmethod
     def from_config(cls, name, config_data, client):
@@ -220,8 +239,8 @@ class Project(object):
 
         return containers
 
-    def stop(self, service_names=None, **options):
+    def stop(self, service_names=None, one_off=OneOffFilter.exclude, **options):
-        containers = self.containers(service_names)
+        containers = self.containers(service_names, one_off=one_off)
 
         def get_deps(container):
             # actually returning inversed dependencies
@@ -249,13 +268,15 @@ class Project(object):
     def kill(self, service_names=None, **options):
         parallel.parallel_kill(self.containers(service_names), options)
 
-    def remove_stopped(self, service_names=None, **options):
+    def remove_stopped(self, service_names=None, one_off=OneOffFilter.exclude, **options):
-        parallel.parallel_remove(self.containers(service_names, stopped=True), options)
+        parallel.parallel_remove(self.containers(
+            service_names, stopped=True, one_off=one_off
+        ), options)
 
     def down(self, remove_image_type, include_volumes, remove_orphans=False):
-        self.stop()
+        self.stop(one_off=OneOffFilter.include)
         self.find_orphan_containers(remove_orphans)
-        self.remove_stopped(v=include_volumes)
+        self.remove_stopped(v=include_volumes, one_off=OneOffFilter.include)
 
         self.networks.remove()
 
@@ -412,7 +433,7 @@ class Project(object):
         for service in self.get_services(service_names, include_deps=False):
             service.pull(ignore_pull_failures)
 
-    def _labeled_containers(self, stopped=False, one_off=False):
+    def _labeled_containers(self, stopped=False, one_off=OneOffFilter.exclude):
         return list(filter(None, [
             Container.from_ps(self.client, container)
             for container in self.client.containers(
@@ -420,7 +441,7 @@ class Project(object):
                 filters={'label': self.labels(one_off=one_off)})])
         )
 
-    def containers(self, service_names=None, stopped=False, one_off=False):
+    def containers(self, service_names=None, stopped=False, one_off=OneOffFilter.exclude):
         if service_names:
             self.validate_service_names(service_names)
         else:

docs/reference/rm.md

@@ -17,6 +17,7 @@ Usage: rm [options] [SERVICE...]
 Options:
 -f, --force   Don't ask to confirm removal
 -v            Remove volumes associated with containers
+-a, --all     Also remove one-off containers
 ```
 
 Removes stopped service containers.

tests/acceptance/cli_test.py

@@ -18,6 +18,7 @@ from docker import errors
 from .. import mock
 from compose.cli.command import get_project
 from compose.container import Container
+from compose.project import OneOffFilter
 from tests.integration.testcases import DockerClientTestCase
 from tests.integration.testcases import get_links
 from tests.integration.testcases import pull_busybox
@@ -105,7 +106,7 @@ class CLITestCase(DockerClientTestCase):
             self.project.kill()
             self.project.remove_stopped()
 
-            for container in self.project.containers(stopped=True, one_off=True):
+            for container in self.project.containers(stopped=True, one_off=OneOffFilter.only):
                 container.remove(force=True)
 
             networks = self.client.networks()
@@ -365,14 +366,22 @@ class CLITestCase(DockerClientTestCase):
     @v2_only()
     def test_down(self):
         self.base_dir = 'tests/fixtures/v2-full'
+
         self.dispatch(['up', '-d'])
         wait_on_condition(ContainerCountCondition(self.project, 2))
 
+        self.dispatch(['run', 'web', 'true'])
+        self.dispatch(['run', '-d', 'web', 'tail', '-f', '/dev/null'])
+        assert len(self.project.containers(one_off=OneOffFilter.only, stopped=True)) == 2
+
         result = self.dispatch(['down', '--rmi=local', '--volumes'])
         assert 'Stopping v2full_web_1' in result.stderr
         assert 'Stopping v2full_other_1' in result.stderr
+        assert 'Stopping v2full_web_run_2' in result.stderr
         assert 'Removing v2full_web_1' in result.stderr
         assert 'Removing v2full_other_1' in result.stderr
+        assert 'Removing v2full_web_run_1' in result.stderr
+        assert 'Removing v2full_web_run_2' in result.stderr
         assert 'Removing volume v2full_data' in result.stderr
         assert 'Removing image v2full_web' in result.stderr
         assert 'Removing image busybox' not in result.stderr
@@ -802,7 +811,7 @@ class CLITestCase(DockerClientTestCase):
         self.assertEqual(len(self.project.containers()), 0)
 
         # Ensure stdin/out was open
-        container = self.project.containers(stopped=True, one_off=True)[0]
+        container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0]
         config = container.inspect()['Config']
         self.assertTrue(config['AttachStderr'])
         self.assertTrue(config['AttachStdout'])
@@ -852,7 +861,7 @@ class CLITestCase(DockerClientTestCase):
 
         self.dispatch(['run', 'implicit'])
         service = self.project.get_service('implicit')
-        containers = service.containers(stopped=True, one_off=True)
+        containers = service.containers(stopped=True, one_off=OneOffFilter.only)
         self.assertEqual(
             [c.human_readable_command for c in containers],
             [u'/bin/sh -c echo "success"'],
@@ -860,7 +869,7 @@ class CLITestCase(DockerClientTestCase):
 
         self.dispatch(['run', 'explicit'])
         service = self.project.get_service('explicit')
-        containers = service.containers(stopped=True, one_off=True)
+        containers = service.containers(stopped=True, one_off=OneOffFilter.only)
         self.assertEqual(
             [c.human_readable_command for c in containers],
             [u'/bin/true'],
@@ -871,7 +880,7 @@ class CLITestCase(DockerClientTestCase):
         name = 'service'
         self.dispatch(['run', '--entrypoint', '/bin/echo', name, 'helloworld'])
         service = self.project.get_service(name)
-        container = service.containers(stopped=True, one_off=True)[0]
+        container = service.containers(stopped=True, one_off=OneOffFilter.only)[0]
         self.assertEqual(
             shlex.split(container.human_readable_command),
             [u'/bin/echo', u'helloworld'],
@@ -883,7 +892,7 @@ class CLITestCase(DockerClientTestCase):
         user = 'sshd'
         self.dispatch(['run', '--user={user}'.format(user=user), name], returncode=1)
         service = self.project.get_service(name)
-        container = service.containers(stopped=True, one_off=True)[0]
+        container = service.containers(stopped=True, one_off=OneOffFilter.only)[0]
         self.assertEqual(user, container.get('Config.User'))
 
     def test_run_service_with_user_overridden_short_form(self):
@@ -892,7 +901,7 @@ class CLITestCase(DockerClientTestCase):
         user = 'sshd'
         self.dispatch(['run', '-u', user, name], returncode=1)
         service = self.project.get_service(name)
-        container = service.containers(stopped=True, one_off=True)[0]
+        container = service.containers(stopped=True, one_off=OneOffFilter.only)[0]
         self.assertEqual(user, container.get('Config.User'))
 
     def test_run_service_with_environement_overridden(self):
@@ -906,7 +915,7 @@ class CLITestCase(DockerClientTestCase):
             '/bin/true',
         ])
         service = self.project.get_service(name)
-        container = service.containers(stopped=True, one_off=True)[0]
+        container = service.containers(stopped=True, one_off=OneOffFilter.only)[0]
         # env overriden
         self.assertEqual('notbar', container.environment['foo'])
         # keep environement from yaml
@@ -920,7 +929,7 @@ class CLITestCase(DockerClientTestCase):
         # create one off container
         self.base_dir = 'tests/fixtures/ports-composefile'
         self.dispatch(['run', '-d', 'simple'])
-        container = self.project.get_service('simple').containers(one_off=True)[0]
+        container = self.project.get_service('simple').containers(one_off=OneOffFilter.only)[0]
 
         # get port information
         port_random = container.get_local_port(3000)
@@ -937,7 +946,7 @@ class CLITestCase(DockerClientTestCase):
         # create one off container
         self.base_dir = 'tests/fixtures/ports-composefile'
         self.dispatch(['run', '-d', '--service-ports', 'simple'])
-        container = self.project.get_service('simple').containers(one_off=True)[0]
+        container = self.project.get_service('simple').containers(one_off=OneOffFilter.only)[0]
 
         # get port information
         port_random = container.get_local_port(3000)
@@ -958,7 +967,7 @@ class CLITestCase(DockerClientTestCase):
         # create one off container
         self.base_dir = 'tests/fixtures/ports-composefile'
         self.dispatch(['run', '-d', '-p', '30000:3000', '--publish', '30001:3001', 'simple'])
-        container = self.project.get_service('simple').containers(one_off=True)[0]
+        container = self.project.get_service('simple').containers(one_off=OneOffFilter.only)[0]
 
         # get port information
         port_short = container.get_local_port(3000)
@@ -980,7 +989,7 @@ class CLITestCase(DockerClientTestCase):
             '--publish', '127.0.0.1:30001:3001',
             'simple'
         ])
-        container = self.project.get_service('simple').containers(one_off=True)[0]
+        container = self.project.get_service('simple').containers(one_off=OneOffFilter.only)[0]
 
         # get port information
         port_short = container.get_local_port(3000)
@@ -997,7 +1006,7 @@ class CLITestCase(DockerClientTestCase):
         # create one off container
         self.base_dir = 'tests/fixtures/expose-composefile'
         self.dispatch(['run', '-d', '--service-ports', 'simple'])
-        container = self.project.get_service('simple').containers(one_off=True)[0]
+        container = self.project.get_service('simple').containers(one_off=OneOffFilter.only)[0]
 
         ports = container.ports
         self.assertEqual(len(ports), 9)
@@ -1021,7 +1030,7 @@ class CLITestCase(DockerClientTestCase):
         self.dispatch(['run', '--name', name, 'service', '/bin/true'])
 
         service = self.project.get_service('service')
-        container, = service.containers(stopped=True, one_off=True)
+        container, = service.containers(stopped=True, one_off=OneOffFilter.only)
         self.assertEqual(container.name, name)
 
     def test_run_service_with_workdir_overridden(self):
@@ -1051,7 +1060,7 @@ class CLITestCase(DockerClientTestCase):
         self.dispatch(['run', 'app', 'nslookup', 'db'])
 
         containers = self.project.get_service('app').containers(
-            stopped=True, one_off=True)
+            stopped=True, one_off=OneOffFilter.only)
         assert len(containers) == 2
 
         for container in containers:
@@ -1071,7 +1080,7 @@ class CLITestCase(DockerClientTestCase):
         self.dispatch(['up', '-d'])
         self.dispatch(['run', '-d', 'app', 'top'])
 
-        container = self.project.get_service('app').containers(one_off=True)[0]
+        container = self.project.get_service('app').containers(one_off=OneOffFilter.only)[0]
         networks = container.get('NetworkSettings.Networks')
 
         assert sorted(list(networks)) == [
@@ -1125,6 +1134,28 @@ class CLITestCase(DockerClientTestCase):
         self.dispatch(['rm', '-f'], None)
         self.assertEqual(len(service.containers(stopped=True)), 0)
 
+    def test_rm_all(self):
+        service = self.project.get_service('simple')
+        service.create_container(one_off=False)
+        service.create_container(one_off=True)
+        kill_service(service)
+        self.assertEqual(len(service.containers(stopped=True)), 1)
+        self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 1)
+        self.dispatch(['rm', '-f'], None)
+        self.assertEqual(len(service.containers(stopped=True)), 0)
+        self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 1)
+        self.dispatch(['rm', '-f', '-a'], None)
+        self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 0)
+
+        service.create_container(one_off=False)
+        service.create_container(one_off=True)
+        kill_service(service)
+        self.assertEqual(len(service.containers(stopped=True)), 1)
+        self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 1)
+        self.dispatch(['rm', '-f', '--all'], None)
+        self.assertEqual(len(service.containers(stopped=True)), 0)
+        self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 0)
+
     def test_stop(self):
         self.dispatch(['up', '-d'], None)
         service = self.project.get_service('simple')

tests/integration/service_test.py

@@ -24,6 +24,7 @@ from compose.const import LABEL_PROJECT
 from compose.const import LABEL_SERVICE
 from compose.const import LABEL_VERSION
 from compose.container import Container
+from compose.project import OneOffFilter
 from compose.service import ConvergencePlan
 from compose.service import ConvergenceStrategy
 from compose.service import NetworkMode
@@ -61,7 +62,7 @@ class ServiceTest(DockerClientTestCase):
         db = self.create_service('db')
         container = db.create_container(one_off=True)
         self.assertEqual(db.containers(stopped=True), [])
-        self.assertEqual(db.containers(one_off=True, stopped=True), [container])
+        self.assertEqual(db.containers(one_off=OneOffFilter.only, stopped=True), [container])
 
     def test_project_is_added_to_container_name(self):
         service = self.create_service('web')
@@ -495,7 +496,7 @@ class ServiceTest(DockerClientTestCase):
         create_and_start_container(db)
         create_and_start_container(db)
 
-        c = create_and_start_container(db, one_off=True)
+        c = create_and_start_container(db, one_off=OneOffFilter.only)
 
         self.assertEqual(
             set(get_links(c)),

tests/unit/service_test.py

@@ -14,6 +14,7 @@ from compose.const import LABEL_ONE_OFF
 from compose.const import LABEL_PROJECT
 from compose.const import LABEL_SERVICE
 from compose.container import Container
+from compose.project import OneOffFilter
 from compose.service import build_ulimits
 from compose.service import build_volume_binding
 from compose.service import BuildAction
@@ -256,7 +257,7 @@ class ServiceTest(unittest.TestCase):
         opts = service._get_container_create_options(
             {'name': name},
             1,
-            one_off=True)
+            one_off=OneOffFilter.only)
         self.assertEqual(opts['name'], name)
 
     def test_get_container_create_options_does_not_mutate_options(self):