diff --git a/changes.go b/changes.go
index 00c9cc7c77..d1b0a25b0d 100644
--- a/changes.go
+++ b/changes.go
@@ -106,50 +106,85 @@ func ChangesAUFS(layers []string, rw string) ([]Change, error) {
 	return changes, nil
 }
 
-func ChangesDirs(newDir, oldDir string) ([]Change, error) {
-	var changes []Change
-	err := filepath.Walk(newDir, func(newPath string, f os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
+type FileInfo struct {
+	parent *FileInfo
+	name string
+	stat syscall.Stat_t
+	children map[string]*FileInfo
+}
 
-		var newStat syscall.Stat_t
-		err = syscall.Lstat(newPath, &newStat)
-		if err != nil {
-			return err
-		}
+func (root *FileInfo) LookUp(path string) *FileInfo {
+	parent := root
+	if path == "/" {
+		return root
+	}
 
-		// Rebase path
-		relPath, err := filepath.Rel(newDir, newPath)
-		if err != nil {
-			return err
-		}
-		relPath = filepath.Join("/", relPath)
-
-		// Skip root
-		if relPath == "/" || relPath == "/.docker-id" {
-			return nil
-		}
-
-		change := Change{
-			Path: relPath,
-		}
-
-		oldPath := filepath.Join(oldDir, relPath)
-
-		var oldStat = &syscall.Stat_t{}
-		err = syscall.Lstat(oldPath, oldStat)
-		if err != nil {
-			if !os.IsNotExist(err) {
-				return err
+	pathElements := strings.Split(path, "/")
+	for _, elem := range pathElements {
+		if elem != "" {
+			child := parent.children[elem]
+			if child == nil {
+				return nil
 			}
-			oldStat = nil
+			parent = child
 		}
+	}
+	return parent
+}
 
-		if oldStat == nil {
-			change.Kind = ChangeAdd
-			changes = append(changes, change)
-		} else {
+func (info *FileInfo)path() string {
+	if info.parent == nil {
+		return "/"
+	}
+	return filepath.Join(info.parent.path(), info.name)
+}
+
+func (info *FileInfo)unlink() {
+	if info.parent != nil {
+		delete(info.parent.children, info.name)
+	}
+}
+
+func (info *FileInfo)Remove(path string) bool {
+	child := info.LookUp(path)
+	if child != nil {
+		child.unlink()
+		return true
+	}
+	return false
+}
+
+func (info *FileInfo)isDir() bool {
+	return info.parent == nil || info.stat.Mode&syscall.S_IFDIR == syscall.S_IFDIR
+}
+
+
+func (info *FileInfo)addChanges(oldInfo *FileInfo, changes *[]Change) {
+	if oldInfo == nil {
+		// add
+		change := Change{
+			Path: info.path(),
+			Kind: ChangeAdd,
+		}
+		*changes = append(*changes, change)
+	}
+
+	// We make a copy so we can modify it to detect additions
+	// also, we only recurse on the old dir if the new info is a directory
+	// otherwise any previous delete/change is considered recursive
+	oldChildren := make(map[string]*FileInfo)
+	if oldInfo != nil && info.isDir() {
+		for k, v := range oldInfo.children {
+			oldChildren[k] = v
+		}
+	}
+
+	for name, newChild := range info.children {
+		oldChild, _ := oldChildren[name]
+		if oldChild != nil {
+			// change?
+			oldStat := &oldChild.stat
+			newStat := &newChild.stat
 			if oldStat.Ino != newStat.Ino ||
 				oldStat.Mode != newStat.Mode ||
 				oldStat.Uid != newStat.Uid ||
@@ -159,50 +194,100 @@ func ChangesDirs(newDir, oldDir string) ([]Change, error) {
 				oldStat.Blocks != newStat.Blocks ||
 				oldStat.Mtim != newStat.Mtim ||
 				oldStat.Ctim != newStat.Ctim {
-				change.Kind = ChangeModify
-				changes = append(changes, change)
+				change := Change{
+					Path: newChild.path(),
+					Kind: ChangeModify,
+				}
+				*changes = append(*changes, change)
 			}
+
+			// Remove from copy so we can detect deletions
+			delete(oldChildren, name)
 		}
 
-		return nil
-	})
-	if err != nil {
-		return nil, err
+		newChild.addChanges(oldChild, changes)
 	}
-	err = filepath.Walk(oldDir, func(oldPath string, f os.FileInfo, err error) error {
+	for _, oldChild := range oldChildren {
+		// delete
+		change := Change{
+			Path: oldChild.path(),
+			Kind: ChangeDelete,
+		}
+		*changes = append(*changes, change)
+	}
+
+
+}
+
+func (info *FileInfo)Changes(oldInfo *FileInfo) []Change {
+	var changes []Change
+
+	info.addChanges(oldInfo, &changes)
+
+	return changes
+}
+
+
+func collectFileInfo(sourceDir string) (*FileInfo, error) {
+	root := &FileInfo {
+		name: "/",
+		children: make(map[string]*FileInfo),
+	}
+
+	err := filepath.Walk(sourceDir, func(path string, f os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
 		// Rebase path
-		relPath, err := filepath.Rel(oldDir, oldPath)
+		relPath, err := filepath.Rel(sourceDir, path)
 		if err != nil {
 			return err
 		}
 		relPath = filepath.Join("/", relPath)
 
-		// Skip root
 		if relPath == "/" {
 			return nil
 		}
 
-		change := Change{
-			Path: relPath,
+		parent := root.LookUp(filepath.Dir(relPath))
+		if parent == nil {
+			return fmt.Errorf("collectFileInfo: Unexpectedly no parent for %s", relPath)
 		}
 
-		newPath := filepath.Join(newDir, relPath)
-
-		var newStat = &syscall.Stat_t{}
-		err = syscall.Lstat(newPath, newStat)
-		if err != nil && os.IsNotExist(err) {
-			change.Kind = ChangeDelete
-			changes = append(changes, change)
+		info := &FileInfo {
+			name: filepath.Base(relPath),
+			children: make(map[string]*FileInfo),
+			parent: parent,
 		}
 
+		if err := syscall.Lstat(path, &info.stat); err != nil {
+			return err
+		}
+
+		parent.children[info.name] = info
+
 		return nil
 	})
 	if err != nil {
 		return nil, err
 	}
-	return changes, nil
+	return root, nil
+}
+
+func ChangesDirs(newDir, oldDir string) ([]Change, error) {
+	oldRoot, err := collectFileInfo(oldDir)
+	if err != nil {
+		return nil, err
+	}
+	newRoot, err := collectFileInfo(newDir)
+	if err != nil {
+		return nil, err
+	}
+
+	// Ignore changes in .docker-id
+	_ = newRoot.Remove("/.docker-id")
+	_ = oldRoot.Remove("/.docker-id")
+
+	return newRoot.Changes(oldRoot), nil
 }