Merge pull request #3841 from alexlarsson/separate-base-fs

Separate out graphdriver mount and container root

buildfile.go

@@ -328,7 +328,7 @@ func (b *buildFile) checkPathForAddition(orig string) error {
 func (b *buildFile) addContext(container *Container, orig, dest string) error {
 	var (
 		origPath = path.Join(b.contextPath, orig)
-		destPath = path.Join(container.RootfsPath(), dest)
+		destPath = path.Join(container.BasefsPath(), dest)
 	)
 	// Preserve the trailing '/'
 	if strings.HasSuffix(dest, "/") {

container.go

@@ -34,7 +34,7 @@ var (
 type Container struct {
 	sync.Mutex
 	root   string // Path to the "home" of the container, including metadata.
-	rootfs string // Path to the root filesystem of the container.
+	basefs string // Path to the graphdriver mountpoint
 
 	ID string
 
@@ -266,7 +266,7 @@ func (container *Container) Inject(file io.Reader, pth string) error {
 	defer container.Unmount()
 
 	// Return error if path exists
-	destPath := path.Join(container.RootfsPath(), pth)
+	destPath := path.Join(container.basefs, pth)
 	if _, err := os.Stat(destPath); err == nil {
 		// Since err is nil, the path could be stat'd and it exists
 		return fmt.Errorf("%s exists", pth)
@@ -278,7 +278,7 @@ func (container *Container) Inject(file io.Reader, pth string) error {
 	}
 
 	// Make sure the directory exists
-	if err := os.MkdirAll(path.Join(container.RootfsPath(), path.Dir(pth)), 0755); err != nil {
+	if err := os.MkdirAll(path.Join(container.basefs, path.Dir(pth)), 0755); err != nil {
 		return err
 	}
 
@@ -707,11 +707,9 @@ func (container *Container) Start() (err error) {
 		return err
 	}
 
-	root := container.RootfsPath()
-
 	if container.Config.WorkingDir != "" {
 		container.Config.WorkingDir = path.Clean(container.Config.WorkingDir)
-		if err := os.MkdirAll(path.Join(root, container.Config.WorkingDir), 0755); err != nil {
+		if err := os.MkdirAll(path.Join(container.basefs, container.Config.WorkingDir), 0755); err != nil {
 			return nil
 		}
 	}
@@ -721,6 +719,23 @@ func (container *Container) Start() (err error) {
 		return err
 	}
 
+	// Setup the root fs as a bind mount of the base fs
+	root := container.RootfsPath()
+	if err := os.MkdirAll(root, 0755); err != nil && !os.IsExist(err) {
+		return nil
+	}
+
+	// Create a bind mount of the base fs as a place where we can add mounts
+	// without affecting the ability to access the base fs
+	if err := mount.Mount(container.basefs, root, "none", "bind,rw"); err != nil {
+		return err
+	}
+
+	// Make sure the root fs is private so the mounts here don't propagate to basefs
+	if err := mount.ForceMount(root, root, "none", "private"); err != nil {
+		return err
+	}
+
 	// Mount docker specific files into the containers root fs
 	if err := mount.Mount(runtime.sysInitPath, path.Join(root, "/.dockerinit"), "none", "bind,ro"); err != nil {
 		return err
@@ -907,8 +922,8 @@ func (container *Container) createVolumes() error {
 		container.VolumesRW[volPath] = srcRW
 
 		// Create the mountpoint
-		volPath = path.Join(container.RootfsPath(), volPath)
+		volPath = path.Join(container.basefs, volPath)
-		rootVolPath, err := utils.FollowSymlinkInScope(volPath, container.RootfsPath())
+		rootVolPath, err := utils.FollowSymlinkInScope(volPath, container.basefs)
 		if err != nil {
 			return err
 		}
@@ -997,7 +1012,7 @@ func (container *Container) applyExternalVolumes() error {
 				if _, exists := container.Volumes[volPath]; exists {
 					continue
 				}
-				if err := os.MkdirAll(path.Join(container.RootfsPath(), volPath), 0755); err != nil {
+				if err := os.MkdirAll(path.Join(container.basefs, volPath), 0755); err != nil {
 					return err
 				}
 				container.Volumes[volPath] = id
@@ -1264,6 +1279,30 @@ func (container *Container) cleanup() {
 		}
 	}
 
+	var (
+		root   = container.RootfsPath()
+		mounts = []string{
+			root,
+			path.Join(root, "/.dockerinit"),
+			path.Join(root, "/.dockerenv"),
+			path.Join(root, "/etc/resolv.conf"),
+		}
+	)
+
+	if container.HostnamePath != "" && container.HostsPath != "" {
+		mounts = append(mounts, path.Join(root, "/etc/hostname"), path.Join(root, "/etc/hosts"))
+	}
+
+	for r := range container.Volumes {
+		mounts = append(mounts, path.Join(root, r))
+	}
+
+	for i := len(mounts) - 1; i >= 0; i-- {
+		if lastError := mount.Unmount(mounts[i]); lastError != nil {
+			log.Printf("Failed to umount %v: %v", mounts[i], lastError)
+		}
+	}
+
 	if err := container.Unmount(); err != nil {
 		log.Printf("%v: Failed to umount filesystem: %v", container.ID, err)
 	}
@@ -1367,7 +1406,7 @@ func (container *Container) Export() (archive.Archive, error) {
 		return nil, err
 	}
 
-	archive, err := archive.Tar(container.RootfsPath(), archive.Uncompressed)
+	archive, err := archive.Tar(container.basefs, archive.Uncompressed)
 	if err != nil {
 		return nil, err
 	}
@@ -1405,32 +1444,6 @@ func (container *Container) GetImage() (*Image, error) {
 }
 
 func (container *Container) Unmount() error {
-	var (
-		err    error
-		root   = container.RootfsPath()
-		mounts = []string{
-			path.Join(root, "/.dockerinit"),
-			path.Join(root, "/.dockerenv"),
-			path.Join(root, "/etc/resolv.conf"),
-		}
-	)
-
-	if container.HostnamePath != "" && container.HostsPath != "" {
-		mounts = append(mounts, path.Join(root, "/etc/hostname"), path.Join(root, "/etc/hosts"))
-	}
-
-	for r := range container.Volumes {
-		mounts = append(mounts, path.Join(root, r))
-	}
-
-	for i := len(mounts) - 1; i >= 0; i-- {
-		if lastError := mount.Unmount(mounts[i]); lastError != nil {
-			err = fmt.Errorf("Failed to umount %v: %v", mounts[i], lastError)
-		}
-	}
-	if err != nil {
-		return err
-	}
 	return container.runtime.Unmount(container)
 }
 
@@ -1467,8 +1480,15 @@ func (container *Container) EnvConfigPath() (string, error) {
 }
 
 // This method must be exported to be used from the lxc template
+// This directory is only usable when the container is running
 func (container *Container) RootfsPath() string {
-	return container.rootfs
+	return path.Join(container.root, "root")
+}
+
+// This is the stand-alone version of the root fs, without any additional mounts.
+// This directory is usable whenever the container is mounted (and not unmounted)
+func (container *Container) BasefsPath() string {
+	return container.basefs
 }
 
 func validateID(id string) error {
@@ -1503,14 +1523,14 @@ func (container *Container) GetSize() (int64, int64) {
 	} else {
 		changes, _ := container.Changes()
 		if changes != nil {
-			sizeRw = archive.ChangesSize(container.RootfsPath(), changes)
+			sizeRw = archive.ChangesSize(container.basefs, changes)
 		} else {
 			sizeRw = -1
 		}
 	}
 
-	if _, err = os.Stat(container.RootfsPath()); err != nil {
+	if _, err = os.Stat(container.basefs); err != nil {
-		if sizeRootfs, err = utils.TreeSize(container.RootfsPath()); err != nil {
+		if sizeRootfs, err = utils.TreeSize(container.basefs); err != nil {
 			sizeRootfs = -1
 		}
 	}
@@ -1522,7 +1542,7 @@ func (container *Container) Copy(resource string) (archive.Archive, error) {
 		return nil, err
 	}
 	var filter []string
-	basePath := path.Join(container.RootfsPath(), resource)
+	basePath := path.Join(container.basefs, resource)
 	stat, err := os.Stat(basePath)
 	if err != nil {
 		container.Unmount()

integration/utils_test.go

@@ -75,7 +75,7 @@ func containerFileExists(eng *engine.Engine, id, dir string, t utils.Fataler) bo
 		t.Fatal(err)
 	}
 	defer c.Unmount()
-	if _, err := os.Stat(path.Join(c.RootfsPath(), dir)); err != nil {
+	if _, err := os.Stat(path.Join(c.BasefsPath(), dir)); err != nil {
 		if os.IsNotExist(err) {
 			return false
 		}

pkg/mount/flags_linux.go

@@ -40,6 +40,7 @@ func parseOptions(options string) (int, string) {
 		"nodiratime":    {false, syscall.MS_NODIRATIME},
 		"bind":          {false, syscall.MS_BIND},
 		"rbind":         {false, syscall.MS_BIND | syscall.MS_REC},
+		"private":       {false, syscall.MS_PRIVATE},
 		"relatime":      {false, syscall.MS_RELATIME},
 		"norelatime":    {true, syscall.MS_RELATIME},
 		"strictatime":   {false, syscall.MS_STRICTATIME},

runtime.go

@@ -132,12 +132,12 @@ func (runtime *Runtime) Register(container *Container) error {
 	}
 
 	// Get the root filesystem from the driver
-	rootfs, err := runtime.driver.Get(container.ID)
+	basefs, err := runtime.driver.Get(container.ID)
 	if err != nil {
 		return fmt.Errorf("Error getting container filesystem %s from driver %s: %s", container.ID, runtime.driver, err)
 	}
 	defer runtime.driver.Put(container.ID)
-	container.rootfs = rootfs
+	container.basefs = basefs
 
 	container.runtime = runtime
 
@@ -765,11 +765,11 @@ func (runtime *Runtime) Mount(container *Container) error {
 	if err != nil {
 		return fmt.Errorf("Error getting container %s from driver %s: %s", container.ID, runtime.driver, err)
 	}
-	if container.rootfs == "" {
+	if container.basefs == "" {
-		container.rootfs = dir
+		container.basefs = dir
-	} else if container.rootfs != dir {
+	} else if container.basefs != dir {
 		return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
-			runtime.driver, container.ID, container.rootfs, dir)
+			runtime.driver, container.ID, container.basefs, dir)
 	}
 	return nil
 }