diff --git a/pkg/archive/diff.go b/pkg/archive/diff.go
index e365f84cc3..1b08ad33ab 100644
--- a/pkg/archive/diff.go
+++ b/pkg/archive/diff.go
@@ -25,6 +25,7 @@ func UnpackLayer(dest string, layer Reader, options *TarOptions) (size int64, er
 	defer pools.BufioReader32KPool.Put(trBuf)
 
 	var dirs []*tar.Header
+	unpackedPaths := make(map[string]struct{})
 
 	if options == nil {
 		options = &TarOptions{}
@@ -134,14 +135,27 @@ func UnpackLayer(dest string, layer Reader, options *TarOptions) (size int64, er
 		if strings.HasPrefix(base, WhiteoutPrefix) {
 			dir := filepath.Dir(path)
 			if base == WhiteoutOpaqueDir {
-				fi, err := os.Lstat(dir)
-				if err != nil && !os.IsNotExist(err) {
+				_, err := os.Lstat(dir)
+				if err != nil {
 					return 0, err
 				}
-				if err := os.RemoveAll(dir); err != nil {
-					return 0, err
-				}
-				if err := os.Mkdir(dir, fi.Mode()&os.ModePerm); err != nil {
+				err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+					if err != nil {
+						if os.IsNotExist(err) {
+							err = nil // parent was deleted
+						}
+						return err
+					}
+					if path == dir {
+						return nil
+					}
+					if _, exists := unpackedPaths[path]; !exists {
+						err := os.RemoveAll(path)
+						return err
+					}
+					return nil
+				})
+				if err != nil {
 					return 0, err
 				}
 			} else {
@@ -214,6 +228,7 @@ func UnpackLayer(dest string, layer Reader, options *TarOptions) (size int64, er
 			if hdr.Typeflag == tar.TypeDir {
 				dirs = append(dirs, hdr)
 			}
+			unpackedPaths[path] = struct{}{}
 		}
 	}
 
diff --git a/pkg/archive/diff_test.go b/pkg/archive/diff_test.go
index 01ed437280..2b29992db5 100644
--- a/pkg/archive/diff_test.go
+++ b/pkg/archive/diff_test.go
@@ -2,7 +2,14 @@ package archive
 
 import (
 	"archive/tar"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
 	"testing"
+
+	"github.com/docker/docker/pkg/ioutils"
 )
 
 func TestApplyLayerInvalidFilenames(t *testing.T) {
@@ -188,3 +195,176 @@ func TestApplyLayerInvalidSymlink(t *testing.T) {
 		}
 	}
 }
+
+func TestApplyLayerWhiteouts(t *testing.T) {
+	wd, err := ioutil.TempDir("", "graphdriver-test-whiteouts")
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(wd)
+
+	base := []string{
+		".baz",
+		"bar/",
+		"bar/bax",
+		"bar/bay/",
+		"baz",
+		"foo/",
+		"foo/.abc",
+		"foo/.bcd/",
+		"foo/.bcd/a",
+		"foo/cde/",
+		"foo/cde/def",
+		"foo/cde/efg",
+		"foo/fgh",
+		"foobar",
+	}
+
+	type tcase struct {
+		change, expected []string
+	}
+
+	tcases := []tcase{
+		{
+			base,
+			base,
+		},
+		{
+			[]string{
+				".bay",
+				".wh.baz",
+				"foo/",
+				"foo/.bce",
+				"foo/.wh..wh..opq",
+				"foo/cde/",
+				"foo/cde/efg",
+			},
+			[]string{
+				".bay",
+				".baz",
+				"bar/",
+				"bar/bax",
+				"bar/bay/",
+				"foo/",
+				"foo/.bce",
+				"foo/cde/",
+				"foo/cde/efg",
+				"foobar",
+			},
+		},
+		{
+			[]string{
+				".bay",
+				".wh..baz",
+				".wh.foobar",
+				"foo/",
+				"foo/.abc",
+				"foo/.wh.cde",
+				"bar/",
+			},
+			[]string{
+				".bay",
+				"bar/",
+				"bar/bax",
+				"bar/bay/",
+				"foo/",
+				"foo/.abc",
+				"foo/.bce",
+			},
+		},
+		{
+			[]string{
+				".abc",
+				".wh..wh..opq",
+				"foobar",
+			},
+			[]string{
+				".abc",
+				"foobar",
+			},
+		},
+	}
+
+	for i, tc := range tcases {
+		l, err := makeTestLayer(tc.change)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = UnpackLayer(wd, l, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = l.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		paths, err := readDirContents(wd)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(tc.expected, paths) {
+			t.Fatalf("invalid files for layer %d: expected %q, got %q", i, tc.expected, paths)
+		}
+	}
+
+}
+
+func makeTestLayer(paths []string) (rc io.ReadCloser, err error) {
+	tmpDir, err := ioutil.TempDir("", "graphdriver-test-mklayer")
+	if err != nil {
+		return
+	}
+	defer func() {
+		if err != nil {
+			os.RemoveAll(tmpDir)
+		}
+	}()
+	for _, p := range paths {
+		if p[len(p)-1] == filepath.Separator {
+			if err = os.MkdirAll(filepath.Join(tmpDir, p), 0700); err != nil {
+				return
+			}
+		} else {
+			if err = ioutil.WriteFile(filepath.Join(tmpDir, p), nil, 0600); err != nil {
+				return
+			}
+		}
+	}
+	archive, err := Tar(tmpDir, Uncompressed)
+	if err != nil {
+		return
+	}
+	return ioutils.NewReadCloserWrapper(archive, func() error {
+		err := archive.Close()
+		os.RemoveAll(tmpDir)
+		return err
+	}), nil
+}
+
+func readDirContents(root string) ([]string, error) {
+	var files []string
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if path == root {
+			return nil
+		}
+		rel, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			rel = rel + "/"
+		}
+		files = append(files, rel)
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return files, nil
+}