Shutdown plugins during daemon shutdown.

Signed-off-by: Anusha Ragunathan <anusha@docker.com>
(cherry picked from commit 863ab9ab134d0baef3c7e5d745eded891e87e734)
Signed-off-by: Tibor Vass <tibor@docker.com>

daemon/daemon.go

@@ -685,6 +685,8 @@ func (daemon *Daemon) Shutdown() error {
 		}
 	}
 
+	pluginShutdown()
+
 	if err := daemon.cleanupMounts(); err != nil {
 		return err
 	}

daemon/daemon_experimental.go

@@ -2,8 +2,15 @@
 
 package daemon
 
-import "github.com/docker/engine-api/types/container"
+import (
+	"github.com/docker/docker/plugin"
+	"github.com/docker/engine-api/types/container"
+)
 
 func (daemon *Daemon) verifyExperimentalContainerSettings(hostConfig *container.HostConfig, config *container.Config) ([]string, error) {
 	return nil, nil
 }
+
+func pluginShutdown() {
+	plugin.GetManager().Shutdown()
+}

daemon/daemon_stub.go

@@ -7,3 +7,6 @@ import "github.com/docker/engine-api/types/container"
 func (daemon *Daemon) verifyExperimentalContainerSettings(hostConfig *container.HostConfig, config *container.Config) ([]string, error) {
 	return nil, nil
 }
+
+func pluginShutdown() {
+}

integration-cli/docker_cli_daemon_experimental_test.go

@@ -5,6 +5,9 @@ package main
 import (
 	"github.com/docker/docker/pkg/integration/checker"
 	"github.com/go-check/check"
+	"os"
+	"os/exec"
+	"time"
 )
 
 var pluginName = "tiborvass/no-remove"
@@ -67,3 +70,66 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithPluginDisabled(c *check.C) {
 	c.Assert(out, checker.Contains, pluginName)
 	c.Assert(out, checker.Contains, "false")
 }
+
+// TestDaemonShutdownLiveRestoreWithPlugins leaves plugin running.
+func (s *DockerDaemonSuite) TestDaemonShutdownLiveRestoreWithPlugins(c *check.C) {
+	if err := s.d.Start("--live-restore"); err != nil {
+		c.Fatalf("Could not start daemon: %v", err)
+	}
+	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName); err != nil {
+		c.Fatalf("Could not install plugin: %v %s", err, out)
+	}
+	defer func() {
+		if err := s.d.Restart("--live-restore"); err != nil {
+			c.Fatalf("Could not restart daemon: %v", err)
+		}
+		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {
+			c.Fatalf("Could not disable plugin: %v %s", err, out)
+		}
+		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {
+			c.Fatalf("Could not remove plugin: %v %s", err, out)
+		}
+	}()
+
+	if err := s.d.Kill(); err != nil {
+		c.Fatalf("Could not kill daemon: %v", err)
+	}
+
+	cmd := exec.Command("pgrep", "-f", "plugin-no-remove")
+	if out, ec, err := runCommandWithOutput(cmd); ec != 0 {
+		c.Fatalf("Expected exit code '0', got %d err: %v output: %s ", ec, err, out)
+	}
+}
+
+// TestDaemonShutdownWithPlugins shuts down running plugins.
+func (s *DockerDaemonSuite) TestDaemonShutdownWithPlugins(c *check.C) {
+	if err := s.d.Start(); err != nil {
+		c.Fatalf("Could not start daemon: %v", err)
+	}
+	if out, err := s.d.Cmd("plugin", "install", "--grant-all-permissions", pluginName); err != nil {
+		c.Fatalf("Could not install plugin: %v %s", err, out)
+	}
+
+	defer func() {
+		if err := s.d.Restart(); err != nil {
+			c.Fatalf("Could not restart daemon: %v", err)
+		}
+		if out, err := s.d.Cmd("plugin", "disable", pluginName); err != nil {
+			c.Fatalf("Could not disable plugin: %v %s", err, out)
+		}
+		if out, err := s.d.Cmd("plugin", "remove", pluginName); err != nil {
+			c.Fatalf("Could not remove plugin: %v %s", err, out)
+		}
+	}()
+
+	if err := s.d.cmd.Process.Signal(os.Interrupt); err != nil {
+		c.Fatalf("Could not kill daemon: %v", err)
+	}
+
+	time.Sleep(5 * time.Second)
+
+	cmd := exec.Command("pgrep", "-f", "plugin-no-remove")
+	if out, ec, err := runCommandWithOutput(cmd); ec != 1 {
+		c.Fatalf("Expected exit code '1', got %d err: %v output: %s ", ec, err, out)
+	}
+}

plugin/manager.go

@@ -47,6 +47,7 @@ type plugin struct {
 	client            *plugins.Client
 	restartManager    restartmanager.RestartManager
 	runtimeSourcePath string
+	exitChan          chan bool
 }
 
 func (p *plugin) Client() *plugins.Client {
@@ -98,6 +99,7 @@ type Manager struct {
 	registryService  registry.Service
 	handleLegacy     bool
 	liveRestore      bool
+	shutdown         bool
 }
 
 // GetManager returns the singleton plugin Manager
@@ -250,10 +252,23 @@ func LookupWithCapability(name, capability string) (Plugin, error) {
 	return nil, ErrInadequateCapability{name, capability}
 }
 
-// StateChanged updates daemon inter...
+// StateChanged updates plugin internals using from libcontainerd events.
 func (pm *Manager) StateChanged(id string, e libcontainerd.StateInfo) error {
 	logrus.Debugf("plugin state changed %s %#v", id, e)
 
+	switch e.State {
+	case libcontainerd.StateExit:
+		pm.RLock()
+		p, idOk := pm.plugins[id]
+		pm.RUnlock()
+		if !idOk {
+			return ErrNotFound(id)
+		}
+		if pm.shutdown == true {
+			p.exitChan <- true
+		}
+	}
+
 	return nil
 }
 

plugin/manager_linux.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"syscall"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/libcontainerd"
@@ -128,3 +129,39 @@ func (pm *Manager) disable(p *plugin) error {
 	pm.save()
 	return nil
 }
+
+// Shutdown stops all plugins and called during daemon shutdown.
+func (pm *Manager) Shutdown() {
+	pm.RLock()
+	defer pm.RUnlock()
+
+	pm.shutdown = true
+	for _, p := range pm.plugins {
+		if p.restartManager != nil {
+			if err := p.restartManager.Cancel(); err != nil {
+				logrus.Error(err)
+			}
+		}
+		if pm.containerdClient != nil {
+			p.exitChan = make(chan bool)
+			err := pm.containerdClient.Signal(p.P.ID, int(syscall.SIGTERM))
+			if err != nil {
+				logrus.Errorf("Sending SIGTERM to plugin failed with error: %v", err)
+			} else {
+				select {
+				case <-p.exitChan:
+					logrus.Debug("Clean shutdown of plugin")
+				case <-time.After(time.Second * 10):
+					logrus.Debug("Force shutdown plugin")
+					if err := pm.containerdClient.Signal(p.P.ID, int(syscall.SIGKILL)); err != nil {
+						logrus.Errorf("Sending SIGKILL to plugin failed with error: %v", err)
+					}
+				}
+			}
+			close(p.exitChan)
+		}
+		if err := os.RemoveAll(p.runtimeSourcePath); err != nil {
+			logrus.Errorf("Remove plugin runtime failed with error: %v", err)
+		}
+	}
+}

plugin/manager_windows.go

@@ -23,3 +23,7 @@ func (pm *Manager) disable(p *plugin) error {
 func (pm *Manager) restore(p *plugin) error {
 	return fmt.Errorf("Not implemented")
 }
+
+// Shutdown plugins
+func (pm *Manager) Shutdown() {
+}