Merge pull request #25142 from tiborvass/1.12.0-final-cherry-picks

1.12.0 final cherry picks

builder/dockerfile/builder.go

@@ -71,6 +71,7 @@ type Builder struct {
 	disableCommit    bool
 	cacheBusted      bool
 	allowedBuildArgs map[string]bool // list of build-time args that are allowed for expansion/substitution and passing to commands in 'run'.
+	directive        parser.Directive
 
 	// TODO: remove once docker.Commit can receive a tag
 	id string
@@ -130,9 +131,15 @@ func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, back
 		tmpContainers:    map[string]struct{}{},
 		id:               stringid.GenerateNonCryptoID(),
 		allowedBuildArgs: make(map[string]bool),
+		directive: parser.Directive{
+			EscapeSeen:           false,
+			LookingForDirectives: true,
+		},
 	}
+	parser.SetEscapeToken(parser.DefaultEscapeToken, &b.directive) // Assume the default token for escape
+
 	if dockerfile != nil {
-		b.dockerfile, err = parser.Parse(dockerfile)
+		b.dockerfile, err = parser.Parse(dockerfile, &b.directive)
 		if err != nil {
 			return nil, err
 		}
@@ -218,7 +225,7 @@ func (b *Builder) build(stdout io.Writer, stderr io.Writer, out io.Writer) (stri
 		for k, v := range b.options.Labels {
 			line += fmt.Sprintf("%q=%q ", k, v)
 		}
-		_, node, err := parser.ParseLine(line)
+		_, node, err := parser.ParseLine(line, &b.directive)
 		if err != nil {
 			return "", err
 		}
@@ -291,7 +298,12 @@ func (b *Builder) Cancel() {
 //
 // TODO: Remove?
 func BuildFromConfig(config *container.Config, changes []string) (*container.Config, error) {
-	ast, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n")))
+	b, err := NewBuilder(context.Background(), nil, nil, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	ast, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n")), &b.directive)
 	if err != nil {
 		return nil, err
 	}
@@ -303,10 +315,6 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
 		}
 	}
 
-	b, err := NewBuilder(context.Background(), nil, nil, nil, nil)
-	if err != nil {
-		return nil, err
-	}
 	b.runConfig = config
 	b.Stdout = ioutil.Discard
 	b.Stderr = ioutil.Discard

builder/dockerfile/evaluator_test.go

@@ -173,7 +173,9 @@ func executeTestCase(t *testing.T, testCase dispatchTestCase) {
 	}()
 
 	r := strings.NewReader(testCase.dockerfile)
-	n, err := parser.Parse(r)
+	d := parser.Directive{}
+	parser.SetEscapeToken(parser.DefaultEscapeToken, &d)
+	n, err := parser.Parse(r, &d)
 
 	if err != nil {
 		t.Fatalf("Error when parsing Dockerfile: %s", err)

builder/dockerfile/internals.go

@@ -427,7 +427,7 @@ func (b *Builder) processImageFrom(img builder.Image) error {
 
 	// parse the ONBUILD triggers by invoking the parser
 	for _, step := range onBuildTriggers {
-		ast, err := parser.Parse(strings.NewReader(step))
+		ast, err := parser.Parse(strings.NewReader(step), &b.directive)
 		if err != nil {
 			return err
 		}
@@ -648,7 +648,7 @@ func (b *Builder) parseDockerfile() error {
 			return fmt.Errorf("The Dockerfile (%s) cannot be empty", b.options.Dockerfile)
 		}
 	}
-	b.dockerfile, err = parser.Parse(f)
+	b.dockerfile, err = parser.Parse(f, &b.directive)
 	if err != nil {
 		return err
 	}

builder/dockerfile/parser/dumper/main.go

@@ -22,7 +22,10 @@ func main() {
 			panic(err)
 		}
 
-		ast, err := parser.Parse(f)
+		d := parser.Directive{LookingForDirectives: true}
+		parser.SetEscapeToken(parser.DefaultEscapeToken, &d)
+
+		ast, err := parser.Parse(f, &d)
 		if err != nil {
 			panic(err)
 		} else {

builder/dockerfile/parser/json_test.go

@@ -28,7 +28,10 @@ var validJSONArraysOfStrings = map[string][]string{
 
 func TestJSONArraysOfStrings(t *testing.T) {
 	for json, expected := range validJSONArraysOfStrings {
-		if node, _, err := parseJSON(json); err != nil {
+		d := Directive{}
+		SetEscapeToken(DefaultEscapeToken, &d)
+
+		if node, _, err := parseJSON(json, &d); err != nil {
 			t.Fatalf("%q should be a valid JSON array of strings, but wasn't! (err: %q)", json, err)
 		} else {
 			i := 0
@@ -48,7 +51,10 @@ func TestJSONArraysOfStrings(t *testing.T) {
 		}
 	}
 	for _, json := range invalidJSONArraysOfStrings {
-		if _, _, err := parseJSON(json); err != errDockerfileNotStringArray {
+		d := Directive{}
+		SetEscapeToken(DefaultEscapeToken, &d)
+
+		if _, _, err := parseJSON(json, &d); err != errDockerfileNotStringArray {
 			t.Fatalf("%q should be an invalid JSON array of strings, but wasn't!", json)
 		}
 	}

builder/dockerfile/parser/line_parsers.go

@@ -21,7 +21,7 @@ var (
 
 // ignore the current argument. This will still leave a command parsed, but
 // will not incorporate the arguments into the ast.
-func parseIgnore(rest string) (*Node, map[string]bool, error) {
+func parseIgnore(rest string, d *Directive) (*Node, map[string]bool, error) {
 	return &Node{}, nil, nil
 }
 
@@ -30,12 +30,12 @@ func parseIgnore(rest string) (*Node, map[string]bool, error) {
 //
 // ONBUILD RUN foo bar -> (onbuild (run foo bar))
 //
-func parseSubCommand(rest string) (*Node, map[string]bool, error) {
+func parseSubCommand(rest string, d *Directive) (*Node, map[string]bool, error) {
 	if rest == "" {
 		return nil, nil, nil
 	}
 
-	_, child, err := ParseLine(rest)
+	_, child, err := ParseLine(rest, d)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -46,7 +46,7 @@ func parseSubCommand(rest string) (*Node, map[string]bool, error) {
 // helper to parse words (i.e space delimited or quoted strings) in a statement.
 // The quotes are preserved as part of this function and they are stripped later
 // as part of processWords().
-func parseWords(rest string) []string {
+func parseWords(rest string, d *Directive) []string {
 	const (
 		inSpaces = iota // looking for start of a word
 		inWord
@@ -96,7 +96,7 @@ func parseWords(rest string) []string {
 				blankOK = true
 				phase = inQuote
 			}
-			if ch == tokenEscape {
+			if ch == d.EscapeToken {
 				if pos+chWidth == len(rest) {
 					continue // just skip an escape token at end of line
 				}
@@ -115,7 +115,7 @@ func parseWords(rest string) []string {
 				phase = inWord
 			}
 			// The escape token is special except for ' quotes - can't escape anything for '
-			if ch == tokenEscape && quote != '\'' {
+			if ch == d.EscapeToken && quote != '\'' {
 				if pos+chWidth == len(rest) {
 					phase = inWord
 					continue // just skip the escape token at end
@@ -133,14 +133,14 @@ func parseWords(rest string) []string {
 
 // parse environment like statements. Note that this does *not* handle
 // variable interpolation, which will be handled in the evaluator.
-func parseNameVal(rest string, key string) (*Node, map[string]bool, error) {
+func parseNameVal(rest string, key string, d *Directive) (*Node, map[string]bool, error) {
 	// This is kind of tricky because we need to support the old
 	// variant:   KEY name value
 	// as well as the new one:    KEY name=value ...
 	// The trigger to know which one is being used will be whether we hit
 	// a space or = first.  space ==> old, "=" ==> new
 
-	words := parseWords(rest)
+	words := parseWords(rest, d)
 	if len(words) == 0 {
 		return nil, nil, nil
 	}
@@ -187,12 +187,12 @@ func parseNameVal(rest string, key string) (*Node, map[string]bool, error) {
 	return rootnode, nil, nil
 }
 
-func parseEnv(rest string) (*Node, map[string]bool, error) {
+func parseEnv(rest string, d *Directive) (*Node, map[string]bool, error) {
-	return parseNameVal(rest, "ENV")
+	return parseNameVal(rest, "ENV", d)
 }
 
-func parseLabel(rest string) (*Node, map[string]bool, error) {
+func parseLabel(rest string, d *Directive) (*Node, map[string]bool, error) {
-	return parseNameVal(rest, "LABEL")
+	return parseNameVal(rest, "LABEL", d)
 }
 
 // parses a statement containing one or more keyword definition(s) and/or
@@ -203,8 +203,8 @@ func parseLabel(rest string) (*Node, map[string]bool, error) {
 // In addition, a keyword definition alone is of the form `keyword` like `name1`
 // above. And the assignments `name2=` and `name3=""` are equivalent and
 // assign an empty value to the respective keywords.
-func parseNameOrNameVal(rest string) (*Node, map[string]bool, error) {
+func parseNameOrNameVal(rest string, d *Directive) (*Node, map[string]bool, error) {
-	words := parseWords(rest)
+	words := parseWords(rest, d)
 	if len(words) == 0 {
 		return nil, nil, nil
 	}
@@ -229,7 +229,7 @@ func parseNameOrNameVal(rest string) (*Node, map[string]bool, error) {
 
 // parses a whitespace-delimited set of arguments. The result is effectively a
 // linked list of string arguments.
-func parseStringsWhitespaceDelimited(rest string) (*Node, map[string]bool, error) {
+func parseStringsWhitespaceDelimited(rest string, d *Directive) (*Node, map[string]bool, error) {
 	if rest == "" {
 		return nil, nil, nil
 	}
@@ -253,7 +253,7 @@ func parseStringsWhitespaceDelimited(rest string) (*Node, map[string]bool, error
 }
 
 // parsestring just wraps the string in quotes and returns a working node.
-func parseString(rest string) (*Node, map[string]bool, error) {
+func parseString(rest string, d *Directive) (*Node, map[string]bool, error) {
 	if rest == "" {
 		return nil, nil, nil
 	}
@@ -263,7 +263,7 @@ func parseString(rest string) (*Node, map[string]bool, error) {
 }
 
 // parseJSON converts JSON arrays to an AST.
-func parseJSON(rest string) (*Node, map[string]bool, error) {
+func parseJSON(rest string, d *Directive) (*Node, map[string]bool, error) {
 	rest = strings.TrimLeftFunc(rest, unicode.IsSpace)
 	if !strings.HasPrefix(rest, "[") {
 		return nil, nil, fmt.Errorf(`Error parsing "%s" as a JSON array`, rest)
@@ -296,12 +296,12 @@ func parseJSON(rest string) (*Node, map[string]bool, error) {
 // parseMaybeJSON determines if the argument appears to be a JSON array. If
 // so, passes to parseJSON; if not, quotes the result and returns a single
 // node.
-func parseMaybeJSON(rest string) (*Node, map[string]bool, error) {
+func parseMaybeJSON(rest string, d *Directive) (*Node, map[string]bool, error) {
 	if rest == "" {
 		return nil, nil, nil
 	}
 
-	node, attrs, err := parseJSON(rest)
+	node, attrs, err := parseJSON(rest, d)
 
 	if err == nil {
 		return node, attrs, nil
@@ -318,8 +318,8 @@ func parseMaybeJSON(rest string) (*Node, map[string]bool, error) {
 // parseMaybeJSONToList determines if the argument appears to be a JSON array. If
 // so, passes to parseJSON; if not, attempts to parse it as a whitespace
 // delimited string.
-func parseMaybeJSONToList(rest string) (*Node, map[string]bool, error) {
+func parseMaybeJSONToList(rest string, d *Directive) (*Node, map[string]bool, error) {
-	node, attrs, err := parseJSON(rest)
+	node, attrs, err := parseJSON(rest, d)
 
 	if err == nil {
 		return node, attrs, nil
@@ -328,11 +328,11 @@ func parseMaybeJSONToList(rest string) (*Node, map[string]bool, error) {
 		return nil, nil, err
 	}
 
-	return parseStringsWhitespaceDelimited(rest)
+	return parseStringsWhitespaceDelimited(rest, d)
 }
 
 // The HEALTHCHECK command is like parseMaybeJSON, but has an extra type argument.
-func parseHealthConfig(rest string) (*Node, map[string]bool, error) {
+func parseHealthConfig(rest string, d *Directive) (*Node, map[string]bool, error) {
 	// Find end of first argument
 	var sep int
 	for ; sep < len(rest); sep++ {
@@ -352,7 +352,7 @@ func parseHealthConfig(rest string) (*Node, map[string]bool, error) {
 	}
 
 	typ := rest[:sep]
-	cmd, attrs, err := parseMaybeJSON(rest[next:])
+	cmd, attrs, err := parseMaybeJSON(rest[next:], d)
 	if err != nil {
 		return nil, nil, err
 	}

builder/dockerfile/parser/parser.go

@@ -36,26 +36,32 @@ type Node struct {
 	EndLine    int             // the line in the original dockerfile where the node ends
 }
 
+// Directive is the structure used during a build run to hold the state of
+// parsing directives.
+type Directive struct {
+	EscapeToken           rune           // Current escape token
+	LineContinuationRegex *regexp.Regexp // Current line contination regex
+	LookingForDirectives  bool           // Whether we are currently looking for directives
+	EscapeSeen            bool           // Whether the escape directive has been seen
+}
+
 var (
-	dispatch              map[string]func(string) (*Node, map[string]bool, error)
+	dispatch           map[string]func(string, *Directive) (*Node, map[string]bool, error)
 	tokenWhitespace    = regexp.MustCompile(`[\t\v\f\r ]+`)
-	tokenLineContinuation *regexp.Regexp
-	tokenEscape           rune
 	tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
 	tokenComment       = regexp.MustCompile(`^#.*$`)
-	lookingForDirectives  bool
-	directiveEscapeSeen   bool
 )
 
-const defaultTokenEscape = "\\"
+// DefaultEscapeToken is the default escape token
+const DefaultEscapeToken = "\\"
 
-// setTokenEscape sets the default token for escaping characters in a Dockerfile.
+// SetEscapeToken sets the default token for escaping characters in a Dockerfile.
-func setTokenEscape(s string) error {
+func SetEscapeToken(s string, d *Directive) error {
 	if s != "`" && s != "\\" {
 		return fmt.Errorf("invalid ESCAPE '%s'. Must be ` or \\", s)
 	}
-	tokenEscape = rune(s[0])
+	d.EscapeToken = rune(s[0])
-	tokenLineContinuation = regexp.MustCompile(`\` + s + `[ \t]*$`)
+	d.LineContinuationRegex = regexp.MustCompile(`\` + s + `[ \t]*$`)
 	return nil
 }
 
@@ -66,7 +72,7 @@ func init() {
 	// reformulating the arguments according to the rules in the parser
 	// functions. Errors are propagated up by Parse() and the resulting AST can
 	// be incorporated directly into the existing AST as a next.
-	dispatch = map[string]func(string) (*Node, map[string]bool, error){
+	dispatch = map[string]func(string, *Directive) (*Node, map[string]bool, error){
 		command.Add:         parseMaybeJSONToList,
 		command.Arg:         parseNameOrNameVal,
 		command.Cmd:         parseMaybeJSON,
@@ -89,36 +95,35 @@ func init() {
 }
 
 // ParseLine parse a line and return the remainder.
-func ParseLine(line string) (string, *Node, error) {
+func ParseLine(line string, d *Directive) (string, *Node, error) {
-
 	// Handle the parser directive '# escape=<char>. Parser directives must precede
 	// any builder instruction or other comments, and cannot be repeated.
-	if lookingForDirectives {
+	if d.LookingForDirectives {
 		tecMatch := tokenEscapeCommand.FindStringSubmatch(strings.ToLower(line))
 		if len(tecMatch) > 0 {
-			if directiveEscapeSeen == true {
+			if d.EscapeSeen == true {
 				return "", nil, fmt.Errorf("only one escape parser directive can be used")
 			}
 			for i, n := range tokenEscapeCommand.SubexpNames() {
 				if n == "escapechar" {
-					if err := setTokenEscape(tecMatch[i]); err != nil {
+					if err := SetEscapeToken(tecMatch[i], d); err != nil {
 						return "", nil, err
 					}
-					directiveEscapeSeen = true
+					d.EscapeSeen = true
 					return "", nil, nil
 				}
 			}
 		}
 	}
 
-	lookingForDirectives = false
+	d.LookingForDirectives = false
 
 	if line = stripComments(line); line == "" {
 		return "", nil, nil
 	}
 
-	if tokenLineContinuation.MatchString(line) {
+	if d.LineContinuationRegex.MatchString(line) {
-		line = tokenLineContinuation.ReplaceAllString(line, "")
+		line = d.LineContinuationRegex.ReplaceAllString(line, "")
 		return line, nil, nil
 	}
 
@@ -130,7 +135,7 @@ func ParseLine(line string) (string, *Node, error) {
 	node := &Node{}
 	node.Value = cmd
 
-	sexp, attrs, err := fullDispatch(cmd, args)
+	sexp, attrs, err := fullDispatch(cmd, args, d)
 	if err != nil {
 		return "", nil, err
 	}
@@ -145,10 +150,7 @@ func ParseLine(line string) (string, *Node, error) {
 
 // Parse is the main parse routine.
 // It handles an io.ReadWriteCloser and returns the root of the AST.
-func Parse(rwc io.Reader) (*Node, error) {
+func Parse(rwc io.Reader, d *Directive) (*Node, error) {
-	directiveEscapeSeen = false
-	lookingForDirectives = true
-	setTokenEscape(defaultTokenEscape) // Assume the default token for escape
 	currentLine := 0
 	root := &Node{}
 	root.StartLine = -1
@@ -163,7 +165,7 @@ func Parse(rwc io.Reader) (*Node, error) {
 		}
 		scannedLine := strings.TrimLeftFunc(string(scannedBytes), unicode.IsSpace)
 		currentLine++
-		line, child, err := ParseLine(scannedLine)
+		line, child, err := ParseLine(scannedLine, d)
 		if err != nil {
 			return nil, err
 		}
@@ -178,7 +180,7 @@ func Parse(rwc io.Reader) (*Node, error) {
 					continue
 				}
 
-				line, child, err = ParseLine(line + newline)
+				line, child, err = ParseLine(line+newline, d)
 				if err != nil {
 					return nil, err
 				}
@@ -188,7 +190,7 @@ func Parse(rwc io.Reader) (*Node, error) {
 				}
 			}
 			if child == nil && line != "" {
-				_, child, err = ParseLine(line)
+				_, child, err = ParseLine(line, d)
 				if err != nil {
 					return nil, err
 				}

builder/dockerfile/parser/parser_test.go

@@ -39,7 +39,9 @@ func TestTestNegative(t *testing.T) {
 			t.Fatalf("Dockerfile missing for %s: %v", dir, err)
 		}
 
-		_, err = Parse(df)
+		d := Directive{LookingForDirectives: true}
+		SetEscapeToken(DefaultEscapeToken, &d)
+		_, err = Parse(df, &d)
 		if err == nil {
 			t.Fatalf("No error parsing broken dockerfile for %s", dir)
 		}
@@ -59,7 +61,9 @@ func TestTestData(t *testing.T) {
 		}
 		defer df.Close()
 
-		ast, err := Parse(df)
+		d := Directive{LookingForDirectives: true}
+		SetEscapeToken(DefaultEscapeToken, &d)
+		ast, err := Parse(df, &d)
 		if err != nil {
 			t.Fatalf("Error parsing %s's dockerfile: %v", dir, err)
 		}
@@ -119,13 +123,15 @@ func TestParseWords(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		words := parseWords(test["input"][0])
+		d := Directive{LookingForDirectives: true}
+		SetEscapeToken(DefaultEscapeToken, &d)
+		words := parseWords(test["input"][0], &d)
 		if len(words) != len(test["expect"]) {
-			t.Fatalf("length check failed. input: %v, expect: %v, output: %v", test["input"][0], test["expect"], words)
+			t.Fatalf("length check failed. input: %v, expect: %q, output: %q", test["input"][0], test["expect"], words)
 		}
 		for i, word := range words {
 			if word != test["expect"][i] {
-				t.Fatalf("word check failed for word: %q. input: %v, expect: %v, output: %v", word, test["input"][0], test["expect"], words)
+				t.Fatalf("word check failed for word: %q. input: %q, expect: %q, output: %q", word, test["input"][0], test["expect"], words)
 			}
 		}
 	}
@@ -138,7 +144,9 @@ func TestLineInformation(t *testing.T) {
 	}
 	defer df.Close()
 
-	ast, err := Parse(df)
+	d := Directive{LookingForDirectives: true}
+	SetEscapeToken(DefaultEscapeToken, &d)
+	ast, err := Parse(df, &d)
 	if err != nil {
 		t.Fatalf("Error parsing dockerfile %s: %v", testFileLineInfo, err)
 	}

builder/dockerfile/parser/utils.go

@@ -36,7 +36,7 @@ func (node *Node) Dump() string {
 
 // performs the dispatch based on the two primal strings, cmd and args. Please
 // look at the dispatch table in parser.go to see how these dispatchers work.
-func fullDispatch(cmd, args string) (*Node, map[string]bool, error) {
+func fullDispatch(cmd, args string, d *Directive) (*Node, map[string]bool, error) {
 	fn := dispatch[cmd]
 
 	// Ignore invalid Dockerfile instructions
@@ -44,7 +44,7 @@ func fullDispatch(cmd, args string) (*Node, map[string]bool, error) {
 		fn = parseIgnore
 	}
 
-	sexp, attrs, err := fn(args)
+	sexp, attrs, err := fn(args, d)
 	if err != nil {
 		return nil, nil, err
 	}

contrib/completion/bash/docker

@@ -1741,6 +1741,7 @@ _docker_service_update() {
 
 	if [ "$subcommand" = "create" ] ; then
 		options_with_args="$options_with_args
+			--container-label
 			--mode
 		"
 
@@ -1754,6 +1755,8 @@ _docker_service_update() {
 	if [ "$subcommand" = "update" ] ; then
 		options_with_args="$options_with_args
 			--arg
+			--container-label-add
+			--container-label-rm
 			--image
 		"
 
@@ -1814,7 +1817,6 @@ _docker_service_update() {
 _docker_swarm() {
 	local subcommands="
 		init
-		inspect
 		join
 		join-token
 		leave
@@ -1855,20 +1857,6 @@ _docker_swarm_init() {
 	esac
 }
 
-_docker_swarm_inspect() {
-	case "$prev" in
-		--format|-f)
-			return
-			;;
-	esac
-
-	case "$cur" in
-		-*)
-			COMPREPLY=( $( compgen -W "--format -f --help" -- "$cur" ) )
-			;;
-	esac
-}
-
 _docker_swarm_join() {
 	case "$prev" in
 		--token)

contrib/init/systemd/docker.service.rpm

@@ -2,7 +2,6 @@
 Description=Docker Application Container Engine
 Documentation=https://docs.docker.com
 After=network.target
-Requires=docker.socket
 
 [Service]
 Type=notify

daemon/graphdriver/graphtest/graphtest_unix.go

@@ -199,6 +199,7 @@ func DriverTestDiffApply(t testing.TB, fileCount int, drivername string, driverO
 	upper := stringid.GenerateRandomID()
 	deleteFile := "file-remove.txt"
 	deleteFileContent := []byte("This file should get removed in upper!")
+	deleteDir := "var/lib"
 
 	if err := driver.Create(base, "", "", nil); err != nil {
 		t.Fatal(err)
@@ -212,6 +213,10 @@ func DriverTestDiffApply(t testing.TB, fileCount int, drivername string, driverO
 		t.Fatal(err)
 	}
 
+	if err := addDirectory(driver, base, deleteDir); err != nil {
+		t.Fatal(err)
+	}
+
 	if err := driver.Create(upper, base, "", nil); err != nil {
 		t.Fatal(err)
 	}
@@ -220,7 +225,7 @@ func DriverTestDiffApply(t testing.TB, fileCount int, drivername string, driverO
 		t.Fatal(err)
 	}
 
-	if err := removeFile(driver, upper, deleteFile); err != nil {
+	if err := removeAll(driver, upper, deleteFile, deleteDir); err != nil {
 		t.Fatal(err)
 	}
 
@@ -271,6 +276,10 @@ func DriverTestDiffApply(t testing.TB, fileCount int, drivername string, driverO
 	if err := checkFileRemoved(driver, diff, deleteFile); err != nil {
 		t.Fatal(err)
 	}
+
+	if err := checkFileRemoved(driver, diff, deleteDir); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // DriverTestChanges tests computed changes on a layer matches changes made

daemon/graphdriver/graphtest/testutil.go

@@ -78,14 +78,29 @@ func addFile(drv graphdriver.Driver, layer, filename string, content []byte) err
 	return ioutil.WriteFile(path.Join(root, filename), content, 0755)
 }
 
-func removeFile(drv graphdriver.Driver, layer, filename string) error {
+func addDirectory(drv graphdriver.Driver, layer, dir string) error {
 	root, err := drv.Get(layer, "")
 	if err != nil {
 		return err
 	}
 	defer drv.Put(layer)
 
-	return os.Remove(path.Join(root, filename))
+	return os.MkdirAll(path.Join(root, dir), 0755)
+}
+
+func removeAll(drv graphdriver.Driver, layer string, names ...string) error {
+	root, err := drv.Get(layer, "")
+	if err != nil {
+		return err
+	}
+	defer drv.Put(layer)
+
+	for _, filename := range names {
+		if err := os.RemoveAll(path.Join(root, filename)); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func checkFileRemoved(drv graphdriver.Driver, layer, filename string) error {

docs/swarm/swarm-tutorial/rolling-update.md

@@ -29,7 +29,6 @@ update delay:
       --replicas 3 \
       --name redis \
       --update-delay 10s \
-      --update-parallelism 1 \
       redis:3.0.6
 
     0u6a4s31ybk7yw2wyvtikmu50
@@ -37,18 +36,21 @@ update delay:
 
     You configure the rolling update policy at service deployment time.
 
-    The `--update-parallelism` flag configures the number of service tasks that
-    the scheduler can update simultaneously. When updates to individual tasks
-    return a state of `RUNNING` or `FAILED`, the scheduler schedules another
-    task to update until all tasks are updated.
-
     The `--update-delay` flag configures the time delay between updates to a
-    service task or sets of tasks.
+    service task or sets of tasks. You can describe the time `T` as a
+    combination of the number of seconds `Ts`, minutes `Tm`, or hours `Th`. So
+    `10m30s` indicates a 10 minute 30 second delay.
 
-    You can describe the time `T` as a combination of the number of seconds
+    By default the scheduler updates 1 task at a time. You can pass the
-    `Ts`, minutes `Tm`, or hours `Th`. So `10m30s` indicates a 10 minute 30
+    `--update-parallelism` flag to configure the maximum number of service tasks
-    second delay.
+    that the scheduler updates simultaneously.
 
+    By default, when an update to an individual task returns a state of
+    `RUNNING`, the scheduler schedules another task to update until all tasks
+    are updated. If, at any time during an update a task returns `FAILED`, the
+    scheduler pauses the update. You can control the behavior using the
+    `--update-failure-action` flag for `docker service create` or
+    `docker service update`.
 
 3. Inspect the `redis` service:
 
@@ -77,13 +79,15 @@ applies the update to nodes according to the `UpdateConfig` policy:
     redis
     ```
 
-    The scheduler applies rolling updates as follows:
+    The scheduler applies rolling updates as follows by default:
 
-    * Stop the initial number of tasks according to `--update-parallelism`.
+    * Stop the first task.
-    * Schedule updates for the stopped tasks.
+    * Schedule update for the stopped task.
-    * Start the containers for the updated tasks.
+    * Start the container for the updated task.
-    * After an update to a task completes, wait for the specified delay
+    * If the update to a task returns `RUNNING`, wait for the
-    period before stopping the next task.
+    specified delay period then stop the next task.
+    * If, at any time during the update, a task returns `FAILED`, pause the
+    update.
 
 5. Run `docker service inspect --pretty redis` to see the new image in the
 desired state:

hack/vendor.sh

@@ -65,7 +65,7 @@ clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
 clone git github.com/imdario/mergo 0.2.1
 
 #get libnetwork packages
-clone git github.com/docker/libnetwork c7dc6dc476a5f00f9b28efebe591347dd64264fc
+clone git github.com/docker/libnetwork 443b7be96fdf0ed8f65ec92953aa8df4f9a725dc
 clone git github.com/docker/go-events afb2b9f2c23f33ada1a22b03651775fdc65a5089
 clone git github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

libcontainerd/client_linux.go

@@ -563,6 +563,8 @@ func (clnt *client) Restore(containerID string, options ...CreateOption) error {
 		clnt.remote.Lock()
 		return nil
 	}
+	// relock because of the defer
+	clnt.remote.Lock()
 
 	clnt.deleteContainer(containerID)
 

pkg/archive/archive_linux.go

@@ -23,7 +23,8 @@ func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os
 	// convert whiteouts to AUFS format
 	if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 {
 		// we just rename the file and make it normal
-		hdr.Name = WhiteoutPrefix + hdr.Name
+		dir, filename := filepath.Split(hdr.Name)
+		hdr.Name = filepath.Join(dir, WhiteoutPrefix+filename)
 		hdr.Mode = 0600
 		hdr.Typeflag = tar.TypeReg
 		hdr.Size = 0

vendor/src/github.com/docker/libnetwork/sandbox.go

@@ -726,6 +726,12 @@ func (sb *sandbox) restoreOslSandbox() error {
 		joinInfo := ep.joinInfo
 		i := ep.iface
 		ep.Unlock()
+
+		if i == nil {
+			log.Errorf("error restoring endpoint %s for container %s", ep.Name(), sb.ContainerID())
+			continue
+		}
+
 		ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes))
 		if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
 			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))

vendor/src/github.com/docker/libnetwork/sandbox_store.go

@@ -245,6 +245,10 @@ func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) {
 					ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID}
 				}
 			}
+			if _, ok := activeSandboxes[sb.ID()]; ok && err != nil {
+				logrus.Errorf("failed to restore endpoint %s in %s for container %s due to %v", eps.Eid, eps.Nid, sb.ContainerID(), err)
+				continue
+			}
 			heap.Push(&sb.endpoints, ep)
 		}