Merge pull request #25767 from tiborvass/1.12.1-rc2-cherrypicks

1.12.1 rc2 cherrypicks

CHANGELOG.md

@@ -114,6 +114,9 @@ To resolve this:
 After making those changes, run `sudo systemctl daemon-reload`, and `sudo
 systemctl restart docker` to reload changes and (re)start the docker daemon.
 
+**IMPORTANT**: With Docker 1.12, a Linux docker installation now has two
+additional binaries; `dockerd`, and `docker-proxy`. If you have scripts for
+installing docker, please make sure to update them accordingly.
 
 ### Builder
 
@@ -186,6 +189,7 @@ systemctl restart docker` to reload changes and (re)start the docker daemon.
 
 ### Runtime
 
++ Split the userland proxy to a separate binary (`docker-proxy`) [#23312](https://github.com/docker/docker/pull/23312)
 + Add `--live-restore` daemon flag to keep containers running when daemon shuts down, and regain control on startup [#23213](https://github.com/docker/docker/pull/23213)
 + Ability to add OCI-compatible runtimes (via `--add-runtime` daemon flag) and select one with `--runtime` on `create` and `run` [#22983](https://github.com/docker/docker/pull/22983)
 + New `overlay2` graphdriver for Linux 4.0+ with multiple lower directory support [#22126](https://github.com/docker/docker/pull/22126)

contrib/builder/deb/armhf/raspbian-jessie/Dockerfile

@@ -7,6 +7,7 @@ RUN sed -i s/httpredir.debian.org/$APT_MIRROR/g /etc/apt/sources.list
 RUN apt-get update && apt-get install -y apparmor bash-completion btrfs-tools build-essential curl ca-certificates debhelper dh-apparmor dh-systemd git libapparmor-dev libdevmapper-dev libltdl-dev libsqlite3-dev libsystemd-journal-dev --no-install-recommends && rm -rf /var/lib/apt/lists/*
 
 ENV GO_VERSION 1.6.3
+ENV GOARM 6
 RUN curl -fSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-armv6l.tar.gz" | tar xzC /usr/local
 ENV PATH $PATH:/usr/local/go/bin
 

hack/install.sh

@@ -389,10 +389,9 @@ do_install() {
 			}
 
 			if [ "$lsb_dist" = "raspbian" ]; then
-				# Create Raspbian specific systemd unit file, use overlay by default
-				( set -x; $sh_c "mkdir -p /etc/systemd/system" )
-				( set -x; $sh_c "$curl https://raw.githubusercontent.com/docker/docker/master/contrib/init/systemd/docker.service > /etc/systemd/system/docker.service" )
-				( set -x; $sh_c "sed -i 's/dockerd/dockerd --storage-driver overlay/' /etc/systemd/system/docker.service" )
+				# Create Raspbian specific systemd drop-in file, use overlay by default
+				( set -x; $sh_c "mkdir -p /etc/systemd/system/docker.service.d" )
+				( set -x; $sh_c "echo '[Service]\nExecStart=\nExecStart=/usr/bin/dockerd --storage-driver overlay -H fd://' > /etc/systemd/system/docker.service.d/overlay.conf" )
 			else
 				# aufs is preferred over devicemapper; try to ensure the driver is available.
 				if ! grep -q aufs /proc/filesystems && ! $sh_c 'modprobe aufs'; then

hack/make/release-rpm

@@ -33,8 +33,9 @@ fi
 for dir in bundles/$VERSION/build-rpm/*/; do
 	version="$(basename "$dir")"
 	suite="${version##*-}"
+	distro="${version%-*}"
 
-	REPO=$YUMDIR/$release/$suite
+	REPO=$YUMDIR/$release/$distro
 
 	# if the directory does not exist, initialize the yum repo
 	if [[ ! -d $REPO/$suite/Packages ]]; then

hack/vendor.sh

@@ -65,7 +65,7 @@ clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
 clone git github.com/imdario/mergo 0.2.1
 
 #get libnetwork packages
-clone git github.com/docker/libnetwork 24f64a6f9e9cade70e3904df291fb321584b1b4e
+clone git github.com/docker/libnetwork 82fb373e3eaa4e9bbb5b5ac148b0a3a71f80fca6
 clone git github.com/docker/go-events afb2b9f2c23f33ada1a22b03651775fdc65a5089
 clone git github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 clone git github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@@ -139,7 +139,7 @@ clone git github.com/docker/docker-credential-helpers v0.3.0
 clone git github.com/docker/containerd 0ac3cd1be170d180b2baed755e8f0da547ceb267
 
 # cluster
-clone git github.com/docker/swarmkit 3708fb309aacfff321759bcdcc99b0f57806d27f
+clone git github.com/docker/swarmkit bfbec9f2b6a487100a80027f6ee16cc0c646a8f5
 clone git github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
 clone git github.com/gogo/protobuf 43a2e0b1c32252bfbbdf81f7faa7a88fb3fa4028
 clone git github.com/cloudflare/cfssl b895b0549c0ff676f92cf09ba971ae02bb41367b

integration-cli/daemon_swarm.go

@@ -311,3 +311,17 @@ func (d *SwarmDaemon) checkControlAvailable(c *check.C) (interface{}, check.Comm
 	c.Assert(info.LocalNodeState, checker.Equals, swarm.LocalNodeStateActive)
 	return info.ControlAvailable, nil
 }
+
+func (d *SwarmDaemon) cmdRetryOutOfSequence(args ...string) (string, error) {
+	for i := 0; ; i++ {
+		out, err := d.Cmd(args[0], args[1:]...)
+		if err != nil {
+			if strings.Contains(err.Error(), "update out of sequence") {
+				if i < 10 {
+					continue
+				}
+			}
+		}
+		return out, err
+	}
+}

integration-cli/docker_cli_swarm_test.go

@@ -207,13 +207,13 @@ func (s *DockerSwarmSuite) TestSwarmPublishAdd(c *check.C) {
 	out, err = d.Cmd("service", "update", "--publish-add", "80:80", name)
 	c.Assert(err, checker.IsNil)
 
-	out, err = d.Cmd("service", "update", "--publish-add", "80:80", name)
+	out, err = d.cmdRetryOutOfSequence("service", "update", "--publish-add", "80:80", name)
 	c.Assert(err, checker.IsNil)
 
-	out, err = d.Cmd("service", "update", "--publish-add", "80:80", "--publish-add", "80:20", name)
+	out, err = d.cmdRetryOutOfSequence("service", "update", "--publish-add", "80:80", "--publish-add", "80:20", name)
 	c.Assert(err, checker.NotNil)
 
-	out, err = d.Cmd("service", "update", "--publish-add", "80:20", name)
+	out, err = d.cmdRetryOutOfSequence("service", "update", "--publish-add", "80:20", name)
 	c.Assert(err, checker.IsNil)
 
 	out, err = d.Cmd("service", "inspect", "--format", "{{ .Spec.EndpointSpec.Ports }}", name)

plugin/manager_linux.go

@@ -90,7 +90,13 @@ func (pm *Manager) initSpec(p *plugin) (*specs.Spec, error) {
 		if mount.Source != nil {
 			m.Source = *mount.Source
 		}
+
 		if m.Source != "" && m.Type == "bind" {
+			/* Debugging issue #25511: Volumes and other content created under the
+			bind mount should be recursively propagated. rshared, not shared.
+			This could be the reason for EBUSY during removal. Override options
+			with rbind, rshared and see if CI errors are fixed. */
+			m.Options = []string{"rbind", "rshared"}
 			fi, err := os.Lstat(filepath.Join(rootfs, string(os.PathSeparator), m.Destination)) // TODO: followsymlinks
 			if err != nil {
 				return nil, err

vendor/src/github.com/docker/libnetwork/network.go

@@ -1110,8 +1110,8 @@ func (n *network) getSvcRecords(ep *endpoint) []etchosts.Record {
 	epName := ep.Name()
 
 	n.ctrlr.Lock()
+	defer n.ctrlr.Unlock()
 	sr, _ := n.ctrlr.svcRecords[n.id]
-	n.ctrlr.Unlock()
 
 	for h, ip := range sr.svcMap {
 		if strings.Split(h, ".")[0] == epName {

vendor/src/github.com/docker/libnetwork/resolver.go

@@ -255,6 +255,9 @@ func (r *resolver) handleSRVQuery(svc string, query *dns.Msg) (*dns.Msg, error)
 	if err != nil {
 		return nil, err
 	}
+	if len(srv) == 0 {
+		return nil, nil
+	}
 	if len(srv) != len(ip) {
 		return nil, fmt.Errorf("invalid reply for SRV query %s", svc)
 	}

vendor/src/github.com/docker/libnetwork/sandbox.go

@@ -444,16 +444,16 @@ func (sb *sandbox) ResolveService(name string) ([]*net.SRV, []net.IP, error) {
 
 	log.Debugf("Service name To resolve: %v", name)
 
+	// There are DNS implementaions that allow SRV queries for names not in
+	// the format defined by RFC 2782. Hence specific validations checks are
+	// not done
 	parts := strings.Split(name, ".")
 	if len(parts) < 3 {
-		return nil, nil, fmt.Errorf("invalid service name, %s", name)
+		return nil, nil, nil
 	}
 
 	portName := parts[0]
 	proto := parts[1]
-	if proto != "_tcp" && proto != "_udp" {
-		return nil, nil, fmt.Errorf("invalid protocol in service, %s", name)
-	}
 	svcName := strings.Join(parts[2:], ".")
 
 	for _, ep := range sb.getConnectedEndpoints() {

vendor/src/github.com/docker/libnetwork/service_linux.go

@@ -647,7 +647,7 @@ func invokeFWMarker(path string, vip net.IP, fwMark uint32, ingressPorts []*Port
 
 	cmd := &exec.Cmd{
 		Path:   reexec.Self(),
-		Args:   append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, eIP.IP.String()),
+		Args:   append([]string{"fwmarker"}, path, vip.String(), fmt.Sprintf("%d", fwMark), addDelOpt, ingressPortsFile, eIP.String()),
 		Stdout: os.Stdout,
 		Stderr: os.Stderr,
 	}
@@ -719,7 +719,13 @@ func fwMarker() {
 	}
 
 	if addDelOpt == "-A" {
-		ruleParams := strings.Fields(fmt.Sprintf("-m ipvs --ipvs -j SNAT --to-source %s", os.Args[6]))
+		eIP, subnet, err := net.ParseCIDR(os.Args[6])
+		if err != nil {
+			logrus.Errorf("Failed to parse endpoint IP %s: %v", os.Args[6], err)
+			os.Exit(9)
+		}
+
+		ruleParams := strings.Fields(fmt.Sprintf("-m ipvs --ipvs -d %s -j SNAT --to-source %s", subnet, eIP))
 		if !iptables.Exists("nat", "POSTROUTING", ruleParams...) {
 			rule := append(strings.Fields("-t nat -A POSTROUTING"), ruleParams...)
 			rules = append(rules, rule)

vendor/src/github.com/docker/swarmkit/ca/certificates.go

@@ -37,13 +37,14 @@ import (
 const (
 	// Security Strength Equivalence
 	//-----------------------------------
-	//| Key-type |  ECC  |  DH/DSA/RSA  |
-	//|   Node   |  256  |     3072     |
-	//|   Root   |  384  |     7680     |
+	//| ECC  |  DH/DSA/RSA  |
+	//| 256  |     3072     |
+	//| 384  |     7680     |
 	//-----------------------------------
 
 	// RootKeySize is the default size of the root CA key
-	RootKeySize = 384
+	// It would be ideal for the root key to use P-384, but in P-384 is not optimized in go yet :(
+	RootKeySize = 256
 	// RootKeyAlgo defines the default algorithm for the root CA Key
 	RootKeyAlgo = "ecdsa"
 	// PassphraseENVVar defines the environment variable to look for the