overlay: don't trust the "trusted.overlay.opaque" xattr

If the overlay filesystem implements an optimization that landed in
97c684cc911060ba7f97c0925eaf842f159a39e8, and in the mainline kernel in
4.10, directories created in merged directories are marked as opaque by
the kernel to let the kernel know that it needn't bother looking at
other layers when reading the contents of that directory.

This means that, when generating a diff for an upper directory, we can't
treat the presence of an opaque attribute as enough of an indication
that a layer diff needs to include whiteout for a directory of the same
name from a lower layer.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>

pkg/archive/changes_linux.go

@@ -300,7 +300,10 @@ func isENOTDIR(err error) bool {
 // OverlayChanges walks the path rw and determines changes for the files in the path,
 // with respect to the parent layers
 func OverlayChanges(layers []string, rw string) ([]Change, error) {
-	return changes(layers, rw, overlayDeletedFile, nil, overlayLowerContainsWhiteout)
+	dc := func(root, path string, fi os.FileInfo) (string, error) {
+		return overlayDeletedFile(layers, root, path, fi)
+	}
+	return changes(layers, rw, dc, nil, overlayLowerContainsWhiteout)
 }
 
 func overlayLowerContainsWhiteout(root, path string) (bool, error) {
@@ -321,23 +324,72 @@ func overlayLowerContainsWhiteout(root, path string) (bool, error) {
 	return false, nil
 }
 
-func overlayDeletedFile(root, path string, fi os.FileInfo) (string, error) {
+func overlayDeletedFile(layers []string, root, path string, fi os.FileInfo) (string, error) {
+	// If it's a whiteout item, then a file or directory with that name is removed by this layer.
 	if fi.Mode()&os.ModeCharDevice != 0 {
 		s := fi.Sys().(*syscall.Stat_t)
 		if major(s.Rdev) == 0 && minor(s.Rdev) == 0 {
 			return path, nil
 		}
 	}
-	if fi.Mode()&os.ModeDir != 0 {
-		opaque, err := system.Lgetxattr(filepath.Join(root, path), "trusted.overlay.opaque")
-		if err != nil {
+	// After this we only need to pay attention to directories.
+	if !fi.IsDir() {
+		return "", nil
+	}
+	// If the directory isn't marked as opaque, then it's just a normal directory.
+	opaque, err := system.Lgetxattr(filepath.Join(root, path), "trusted.overlay.opaque")
+	if err != nil {
+		return "", err
+	}
+	if len(opaque) != 1 || opaque[0] != 'y' {
+		return "", err
+	}
+	// If there are no lower layers, then it can't have been deleted and recreated in this layer.
+	if len(layers) == 0 {
+		return "", err
+	}
+	// At this point, we have a directory that's opaque.  If it appears in one of the lower
+	// layers, then it was newly-created here, so it wasn't also deleted here.
+	for _, layer := range layers {
+		stat, err := os.Stat(filepath.Join(layer, path))
+		if err != nil && !os.IsNotExist(err) && !isENOTDIR(err) {
+			// Not sure what happened here.
 			return "", err
 		}
-		if len(opaque) == 1 && opaque[0] == 'y' {
+		if err == nil {
+			if stat.Mode()&os.ModeCharDevice != 0 {
+				// It's a whiteout for this directory, so it can't have been
+				// deleted in this layer.
+				s := stat.Sys().(*syscall.Stat_t)
+				if major(s.Rdev) == 0 && minor(s.Rdev) == 0 {
+					return "", nil
+				}
+			}
+			// It's not whiteout, so it was there in the older layer, so it has to be
+			// marked as deleted in this layer.
 			return path, nil
 		}
+		for dir := filepath.Dir(path); dir != "" && dir != string(os.PathSeparator); dir = filepath.Dir(dir) {
+			// Check for whiteout for a parent directory.
+			stat, err := os.Stat(filepath.Join(layer, dir))
+			if err != nil && !os.IsNotExist(err) && !isENOTDIR(err) {
+				// Not sure what happened here.
+				return "", err
+			}
+			if err == nil {
+				if stat.Mode()&os.ModeCharDevice != 0 {
+					// If it's whiteout for a parent directory, then the
+					// original directory wasn't inherited into the top layer.
+					s := stat.Sys().(*syscall.Stat_t)
+					if major(s.Rdev) == 0 && minor(s.Rdev) == 0 {
+						return "", nil
+					}
+				}
+			}
+		}
 	}
 
+	// We didn't find the same path in any older layers, so it was new in this one.
 	return "", nil
 
 }