From b9b66d0e1b70cd0475d6fbba33eba5632f852dde Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 20 Feb 2013 17:45:46 -0800 Subject: [PATCH 01/35] sysinit: Support for the -g (gateway) flag used in networking setup --- sysinit.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sysinit.go b/sysinit.go index 99bf43d25f..f701417978 100644 --- a/sysinit.go +++ b/sysinit.go @@ -11,6 +11,17 @@ import ( "syscall" ) +// Setup networking +func setupNetworking(gw string) { + if gw == "" { + return + } + cmd := exec.Command("/sbin/route", "add", "default", "gw", gw) + if err := cmd.Run(); err != nil { + log.Fatalf("Unable to set up networking: %v", err) + } +} + // Takes care of dropping privileges to the desired user func changeUser(u string) { if u == "" { @@ -62,8 +73,11 @@ func SysInit() { os.Exit(1) } var u = flag.String("u", "", "username or uid") + var gw = flag.String("g", "", "gateway address") flag.Parse() + + setupNetworking(*gw) changeUser(*u) executeProgram(flag.Arg(0), flag.Args()) } From 5cecd548cd48cec8967f7ad0b0b42b30fa3ec7a0 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 20 Feb 2013 17:47:09 -0800 Subject: [PATCH 02/35] Basic networking support with hardcoded addresses. Work in progress. --- container.go | 13 +++++++++++++ lxc_template.go | 14 +++++++------- network.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 network.go diff --git a/container.go b/container.go index 57ec531737..50ac5e715e 100644 --- a/container.go +++ b/container.go @@ -33,6 +33,7 @@ type Container struct { Config *Config Filesystem *Filesystem + Network *NetworkInterface State *State SysInitPath string @@ -87,6 +88,10 @@ func createContainer(id string, root string, command string, args []string, laye if err := container.Filesystem.createMountPoints(); err != nil { return nil, err } + var err error + if container.Network, err = allocateNetwork(); err != nil { + return nil, err + } if err := container.save(); err != nil { return nil, err } @@ -272,11 +277,19 @@ func (container *Container) Start() error { "--", "/sbin/init", } + + // Networking + params = append(params, "-g", container.Network.Gateway.String()) + + // User if container.Config.User != "" { params = append(params, "-u", container.Config.User) } + + // Program params = append(params, "--", container.Path) params = append(params, container.Args...) + container.cmd = exec.Command("/usr/bin/lxc-start", params...) var err error diff --git a/lxc_template.go b/lxc_template.go index 4ac72da273..48e6dc732b 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -14,12 +14,12 @@ lxc.utsname = {{.Id}} #lxc.aa_profile = unconfined # network configuration -#lxc.network.type = veth -#lxc.network.flags = up -#lxc.network.link = br0 -#lxc.network.name = eth0 # Internal container network interface name -#lxc.network.mtu = 1500 -#lxc.network.ipv4 = {ip_address}/{ip_prefix_len} +lxc.network.type = veth +lxc.network.flags = up +lxc.network.link = lxcbr0 +lxc.network.name = eth0 +lxc.network.mtu = 1500 +lxc.network.ipv4 = {{.Network.IpAddress}}/{{.Network.IpPrefixLen}} # root filesystem {{$ROOTFS := .Filesystem.RootFS}} @@ -82,7 +82,7 @@ lxc.mount.entry = /etc/resolv.conf {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 # drop linux capabilities (apply mainly to the user root in the container) -lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw setfcap setpcap sys_admin sys_boot sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config +#lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw setfcap setpcap sys_admin sys_boot sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config # limits {{if .Config.Ram}} diff --git a/network.go b/network.go new file mode 100644 index 0000000000..234086c64c --- /dev/null +++ b/network.go @@ -0,0 +1,29 @@ +package docker + +import ( + "net" +) + +const ( + networkGateway = "10.0.3.1" + networkPrefixLen = 24 +) + +type NetworkInterface struct { + IpAddress string + IpPrefixLen int + Gateway net.IP +} + +func allocateIPAddress() string { + return "10.0.3.2" +} + +func allocateNetwork() (*NetworkInterface, error) { + iface := &NetworkInterface{ + IpAddress: allocateIPAddress(), + IpPrefixLen: networkPrefixLen, + Gateway: net.ParseIP(networkGateway), + } + return iface, nil +} From 5039d4a2804561885e32f2a93cb2a51cbaa8e847 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 20 Feb 2013 18:20:18 -0800 Subject: [PATCH 03/35] Network: Automatically figure out the gateway and netmask by inspecting the lxc bridge interface --- network.go | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/network.go b/network.go index 234086c64c..0a6d1b8598 100644 --- a/network.go +++ b/network.go @@ -1,12 +1,12 @@ package docker import ( + "fmt" "net" ) const ( - networkGateway = "10.0.3.1" - networkPrefixLen = 24 + networkBridgeIface = "lxcbr0" ) type NetworkInterface struct { @@ -19,11 +19,42 @@ func allocateIPAddress() string { return "10.0.3.2" } +func getBridgeAddr(name string) (net.Addr, error) { + iface, err := net.InterfaceByName(name) + if err != nil { + return nil, err + } + addrs, err := iface.Addrs() + if err != nil { + return nil, err + } + var addrs4 []net.Addr + for _, addr := range addrs { + ip := (addr.(*net.IPNet)).IP + if ip4 := ip.To4(); len(ip4) == net.IPv4len { + addrs4 = append(addrs4, addr) + } + } + switch { + case len(addrs4) == 0: + return nil, fmt.Errorf("Bridge %v has no IP addresses", name) + case len(addrs4) > 1: + return nil, fmt.Errorf("Bridge %v has more than 1 IPv4 address", name) + } + return addrs4[0], nil +} + func allocateNetwork() (*NetworkInterface, error) { + bridgeAddr, err := getBridgeAddr(networkBridgeIface) + if err != nil { + return nil, err + } + bridge := bridgeAddr.(*net.IPNet) + ipPrefixLen, _ := bridge.Mask.Size() iface := &NetworkInterface{ IpAddress: allocateIPAddress(), - IpPrefixLen: networkPrefixLen, - Gateway: net.ParseIP(networkGateway), + IpPrefixLen: ipPrefixLen, + Gateway: bridge.IP, } return iface, nil } From 6124c5eb31ab8f9db4db288002388554d6181c86 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Wed, 20 Feb 2013 19:18:01 -0800 Subject: [PATCH 04/35] Network: Simple random IP allocation on the bridge network. --- network.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/network.go b/network.go index 0a6d1b8598..75f4ceae3b 100644 --- a/network.go +++ b/network.go @@ -2,6 +2,7 @@ package docker import ( "fmt" + "math/rand" "net" ) @@ -15,8 +16,10 @@ type NetworkInterface struct { Gateway net.IP } -func allocateIPAddress() string { - return "10.0.3.2" +func allocateIPAddress(network *net.IPNet) net.IP { + ip := network.IP.Mask(network.Mask) + ip[3] = byte(rand.Intn(254)) + return ip } func getBridgeAddr(name string) (net.Addr, error) { @@ -52,7 +55,7 @@ func allocateNetwork() (*NetworkInterface, error) { bridge := bridgeAddr.(*net.IPNet) ipPrefixLen, _ := bridge.Mask.Size() iface := &NetworkInterface{ - IpAddress: allocateIPAddress(), + IpAddress: allocateIPAddress(bridge).String(), IpPrefixLen: ipPrefixLen, Gateway: bridge.IP, } From e0e49b9a2259d779b31055339e67b054c8dffc3b Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 21 Feb 2013 18:33:23 -0800 Subject: [PATCH 05/35] Network: Do not assume that we are using a class C. Instead, compute the IP addresses range and network size in order to allocate an IP address. --- network.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/network.go b/network.go index 75f4ceae3b..1e9a2edd5d 100644 --- a/network.go +++ b/network.go @@ -1,6 +1,8 @@ package docker import ( + "bytes" + "encoding/binary" "fmt" "math/rand" "net" @@ -16,10 +18,61 @@ type NetworkInterface struct { Gateway net.IP } -func allocateIPAddress(network *net.IPNet) net.IP { - ip := network.IP.Mask(network.Mask) - ip[3] = byte(rand.Intn(254)) - return ip +func networkRange(network *net.IPNet) (net.IP, net.IP) { + netIP := network.IP.To4() + firstIP := netIP.Mask(network.Mask) + lastIP := net.IPv4(0, 0, 0, 0).To4() + for i := 0; i < len(lastIP); i++ { + lastIP[i] = netIP[i] | ^network.Mask[i] + } + return firstIP, lastIP +} + +func ipToInt(ip net.IP) (int32, error) { + buf := bytes.NewBuffer(ip.To4()) + var n int32 + if err := binary.Read(buf, binary.BigEndian, &n); err != nil { + return 0, err + } + return n, nil +} + +func intToIp(n int32) (net.IP, error) { + var buf bytes.Buffer + if err := binary.Write(&buf, binary.BigEndian, &n); err != nil { + return net.IP{}, err + } + ip := net.IPv4(0, 0, 0, 0).To4() + for i := 0; i < net.IPv4len; i++ { + ip[i] = buf.Bytes()[i] + } + return ip, nil +} + +func networkSize(mask net.IPMask) (int32, error) { + for i := 0; i < net.IPv4len; i++ { + mask[i] = ^mask[i] + } + buf := bytes.NewBuffer(mask) + var n int32 + if err := binary.Read(buf, binary.BigEndian, &n); err != nil { + return 0, err + } + return n + 1, nil +} + +func allocateIPAddress(network *net.IPNet) (net.IP, error) { + ip, _ := networkRange(network) + netSize, err := networkSize(network.Mask) + if err != nil { + return net.IP{}, err + } + numIp, err := ipToInt(ip) + if err != nil { + return net.IP{}, err + } + numIp += rand.Int31n(netSize) + return intToIp(numIp) } func getBridgeAddr(name string) (net.Addr, error) { @@ -54,8 +107,12 @@ func allocateNetwork() (*NetworkInterface, error) { } bridge := bridgeAddr.(*net.IPNet) ipPrefixLen, _ := bridge.Mask.Size() + ip, err := allocateIPAddress(bridge) + if err != nil { + return nil, err + } iface := &NetworkInterface{ - IpAddress: allocateIPAddress(bridge).String(), + IpAddress: ip.String(), IpPrefixLen: ipPrefixLen, Gateway: bridge.IP, } From 149badc22b45acb171c3e583f6e820a06e4ced87 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 21 Feb 2013 18:34:35 -0800 Subject: [PATCH 06/35] Network tests --- network_test.go | 101 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 network_test.go diff --git a/network_test.go b/network_test.go new file mode 100644 index 0000000000..c1ea382783 --- /dev/null +++ b/network_test.go @@ -0,0 +1,101 @@ +package docker + +import ( + "net" + "testing" +) + +func TestNetworkRange(t *testing.T) { + // Simple class C test + _, network, _ := net.ParseCIDR("192.168.0.1/24") + first, last := networkRange(network) + if !first.Equal(net.ParseIP("192.168.0.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("192.168.0.255")) { + t.Error(last.String()) + } + if size, err := networkSize(network.Mask); err != nil || size != 256 { + t.Error(size, err) + } + + // Class A test + _, network, _ = net.ParseCIDR("10.0.0.1/8") + first, last = networkRange(network) + if !first.Equal(net.ParseIP("10.0.0.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.255.255.255")) { + t.Error(last.String()) + } + if size, err := networkSize(network.Mask); err != nil || size != 16777216 { + t.Error(size, err) + } + + // Class A, random IP address + _, network, _ = net.ParseCIDR("10.1.2.3/8") + first, last = networkRange(network) + if !first.Equal(net.ParseIP("10.0.0.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.255.255.255")) { + t.Error(last.String()) + } + + // 32bit mask + _, network, _ = net.ParseCIDR("10.1.2.3/32") + first, last = networkRange(network) + if !first.Equal(net.ParseIP("10.1.2.3")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.1.2.3")) { + t.Error(last.String()) + } + if size, err := networkSize(network.Mask); err != nil || size != 1 { + t.Error(size, err) + } + + // 31bit mask + _, network, _ = net.ParseCIDR("10.1.2.3/31") + first, last = networkRange(network) + if !first.Equal(net.ParseIP("10.1.2.2")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.1.2.3")) { + t.Error(last.String()) + } + if size, err := networkSize(network.Mask); err != nil || size != 2 { + t.Error(size, err) + } + + // 26bit mask + _, network, _ = net.ParseCIDR("10.1.2.3/26") + first, last = networkRange(network) + if !first.Equal(net.ParseIP("10.1.2.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.1.2.63")) { + t.Error(last.String()) + } + if size, err := networkSize(network.Mask); err != nil || size != 64 { + t.Error(size, err) + } +} + +func TestConversion(t *testing.T) { + ip := net.ParseIP("127.0.0.1") + i, err := ipToInt(ip) + if err != nil { + t.Fatal(err) + } + if i == 0 { + t.Fatal("converted to zero") + } + conv, err := intToIp(i) + if err != nil { + t.Fatal(err) + } + if !ip.Equal(conv) { + t.Error(conv.String()) + } +} From f437f5b8b4a706d56f9a7cfba1dbc6b21e9a33f3 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 22 Feb 2013 12:28:25 -0800 Subject: [PATCH 07/35] 'docker pull' and 'docker put' automatically detect tar compression (gzip, bzip2 or uncompressed). -j and -z flags are no longer required. --- README.md | 4 +-- fake/fake.go | 2 +- filesystem.go | 3 +- future/future.go | 23 ++++++++++++++ image/archive.go | 71 +++++++++++++++++++++++++++++++++++++++++++ image/archive_test.go | 54 ++++++++++++++++++++++++++++++++ image/image.go | 14 +++------ image/layers.go | 63 ++++++++++++++++---------------------- server/server.go | 27 ++++------------ utils.go | 19 ------------ 10 files changed, 190 insertions(+), 90 deletions(-) create mode 100644 image/archive.go create mode 100644 image/archive_test.go diff --git a/README.md b/README.md index 42fd479b70..cf3f74dc61 100644 --- a/README.md +++ b/README.md @@ -157,12 +157,12 @@ Step by step host setup 3. Type the following commands: apt-get update - apt-get install lxc wget + apt-get install lxc wget bsdtar 4. Download the latest version of the [docker binaries](https://dl.dropbox.com/u/20637798/docker.tar.gz) (`wget https://dl.dropbox.com/u/20637798/docker.tar.gz`) (warning: this may not be the most up-to-date build) 5. Extract the contents of the tar file `tar -xf docker.tar.gz` 6. Launch the docker daemon `./dockerd` -7. Download a base image by running 'docker pull -j base' +7. Download a base image by running 'docker pull base' Client installation diff --git a/fake/fake.go b/fake/fake.go index a8c18a458e..c1e694515b 100644 --- a/fake/fake.go +++ b/fake/fake.go @@ -14,7 +14,7 @@ func FakeTar() (io.Reader, error) { content := []byte("Hello world!\n") buf := new(bytes.Buffer) tw := tar.NewWriter(buf) - for _, name := range []string {"/etc/postgres/postgres.conf", "/etc/passwd", "/var/log/postgres", "/var/log/postgres/postgres.conf"} { + for _, name := range []string {"hello", "etc/postgres/postgres.conf", "etc/passwd", "var/log/postgres/postgres.conf"} { hdr := new(tar.Header) hdr.Size = int64(len(content)) hdr.Name = name diff --git a/filesystem.go b/filesystem.go index c775fa0c18..eadc6f69be 100644 --- a/filesystem.go +++ b/filesystem.go @@ -10,6 +10,7 @@ import ( "strings" "syscall" "time" + "github.com/dotcloud/docker/image" ) type Filesystem struct { @@ -104,7 +105,7 @@ func (fs *Filesystem) Tar() (io.Reader, error) { if err := fs.EnsureMounted(); err != nil { return nil, err } - return Tar(fs.RootFS) + return image.Tar(fs.RootFS, image.Uncompressed) } func (fs *Filesystem) EnsureMounted() error { diff --git a/future/future.go b/future/future.go index a0efacc03c..b1427e1007 100644 --- a/future/future.go +++ b/future/future.go @@ -61,3 +61,26 @@ func Go(f func() error) chan error { return ch } +// Pv wraps an io.Reader such that it is passed through unchanged, +// but logs the number of bytes copied (comparable to the unix command pv) +func Pv(src io.Reader, info io.Writer) io.Reader { + var totalBytes int + data := make([]byte, 2048) + r, w := io.Pipe() + go func() { + for { + if n, err := src.Read(data); err != nil { + w.CloseWithError(err) + return + } else { + totalBytes += n + fmt.Fprintf(info, "--> %d bytes\n", totalBytes) + if _, err = w.Write(data[:n]); err != nil { + return + } + } + } + }() + return r +} + diff --git a/image/archive.go b/image/archive.go new file mode 100644 index 0000000000..bc8edb4bca --- /dev/null +++ b/image/archive.go @@ -0,0 +1,71 @@ +package image + +import ( + "io" + "io/ioutil" + "os/exec" + "errors" +) + +type Compression uint32 + +const ( + Uncompressed Compression = iota + Bzip2 + Gzip +) + +func (compression *Compression) Flag() string { + switch *compression { + case Bzip2: return "j" + case Gzip: return "z" + } + return "" +} + +func Tar(path string, compression Compression) (io.Reader, error) { + cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-c" + compression.Flag(), ".") + return CmdStream(cmd) +} + +func Untar(archive io.Reader, path string) error { + cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-x") + cmd.Stdin = archive + output, err := cmd.CombinedOutput() + if err != nil { + return errors.New(err.Error() + ": " + string(output)) + } + return nil +} + +func CmdStream(cmd *exec.Cmd) (io.Reader, error) { + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + pipeR, pipeW := io.Pipe() + go func() { + _, err := io.Copy(pipeW, stdout) + if err != nil { + pipeW.CloseWithError(err) + } + errText, e := ioutil.ReadAll(stderr) + if e != nil { + errText = []byte("(...couldn't fetch stderr: " + e.Error() + ")") + } + if err := cmd.Wait(); err != nil { + // FIXME: can this block if stderr outputs more than the size of StderrPipe()'s buffer? + pipeW.CloseWithError(errors.New(err.Error() + ": " + string(errText))) + } else { + pipeW.Close() + } + }() + if err := cmd.Start(); err != nil { + return nil, err + } + return pipeR, nil +} diff --git a/image/archive_test.go b/image/archive_test.go new file mode 100644 index 0000000000..0c19e605fe --- /dev/null +++ b/image/archive_test.go @@ -0,0 +1,54 @@ +package image + +import ( + "testing" + "os" + "os/exec" + "io/ioutil" +) + +func TestCmdStreamBad(t *testing.T) { + badCmd := exec.Command("/bin/sh", "-c", "echo hello; echo >&2 error couldn\\'t reverse the phase pulser; exit 1") + out, err := CmdStream(badCmd) + if err != nil { + t.Fatalf("Failed to start command: " + err.Error()) + } + if output, err := ioutil.ReadAll(out); err == nil { + t.Fatalf("Command should have failed") + } else if err.Error() != "exit status 1: error couldn't reverse the phase pulser\n" { + t.Fatalf("Wrong error value (%s)", err.Error()) + } else if s := string(output); s != "hello\n" { + t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output) + } +} + +func TestCmdStreamGood(t *testing.T) { + cmd := exec.Command("/bin/sh", "-c", "echo hello; exit 0") + out, err := CmdStream(cmd) + if err != nil { + t.Fatal(err) + } + if output, err := ioutil.ReadAll(out); err != nil { + t.Fatalf("Command should not have failed (err=%s)", err) + } else if s := string(output); s != "hello\n" { + t.Fatalf("Command output should be '%s', not '%s'", "hello\\n", output) + } +} + +func TestTarUntar(t *testing.T) { + archive, err := Tar(".", Uncompressed) + if err != nil { + t.Fatal(err) + } + tmp, err := ioutil.TempDir("", "docker-test-untar") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + if err := Untar(archive, tmp); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(tmp); err != nil { + t.Fatalf("Error stating %s: %s", tmp, err.Error()) + } +} diff --git a/image/image.go b/image/image.go index 82c88b2a05..d819237a7e 100644 --- a/image/image.go +++ b/image/image.go @@ -44,16 +44,10 @@ func New(root string) (*Store, error) { }, nil } -type Compression uint32 - -const ( - Uncompressed Compression = iota - Bzip2 - Gzip -) - -func (store *Store) Import(name string, archive io.Reader, stderr io.Writer, parent *Image, compression Compression) (*Image, error) { - layer, err := store.Layers.AddLayer(archive, stderr, compression) +// Import creates a new image from the contents of `archive` and registers it in the store as `name`. +// If `parent` is not nil, it will registered as the parent of the new image. +func (store *Store) Import(name string, archive io.Reader, parent *Image) (*Image, error) { + layer, err := store.Layers.AddLayer(archive) if err != nil { return nil, err } diff --git a/image/layers.go b/image/layers.go index f856ff81d2..61782cd88c 100644 --- a/image/layers.go +++ b/image/layers.go @@ -7,7 +7,6 @@ import ( "io" "io/ioutil" "os" - "os/exec" "github.com/dotcloud/docker/future" ) @@ -82,50 +81,42 @@ func (store *LayerStore) layerPath(id string) string { } -func (store *LayerStore) AddLayer(archive io.Reader, stderr io.Writer, compression Compression) (string, error) { +func (store *LayerStore) AddLayer(archive io.Reader) (string, error) { + errors := make(chan error) + // Untar tmp, err := store.Mktemp() defer os.RemoveAll(tmp) if err != nil { return "", err } - extractFlags := "-x" - if compression == Bzip2 { - extractFlags += "j" - } else if compression == Gzip { - extractFlags += "z" - } - untarCmd := exec.Command("tar", "-C", tmp, extractFlags) - untarW, err := untarCmd.StdinPipe() - if err != nil { - return "", err - } - untarStderr, err := untarCmd.StderrPipe() - if err != nil { - return "", err - } - go io.Copy(stderr, untarStderr) - untarStdout, err := untarCmd.StdoutPipe() - if err != nil { - return "", err - } - go io.Copy(stderr, untarStdout) - untarCmd.Start() + untarR, untarW := io.Pipe() + go func() { + errors <- Untar(untarR, tmp) + }() + // Compute ID + var id string hashR, hashW := io.Pipe() - job_copy := future.Go(func() error { - _, err := io.Copy(io.MultiWriter(hashW, untarW), archive) - hashW.Close() - untarW.Close() - return err - }) - id, err := future.ComputeId(hashR) + go func() { + _id, err := future.ComputeId(hashR) + id = _id + errors <- err + }() + // Duplicate archive to each stream + _, err = io.Copy(io.MultiWriter(hashW, untarW), archive) + hashW.Close() + untarW.Close() if err != nil { return "", err } - if err := untarCmd.Wait(); err != nil { - return "", err - } - if err := <-job_copy; err != nil { - return "", err + // Wait for goroutines + for i:=0; i<2; i+=1 { + select { + case err := <-errors: { + if err != nil { + return "", err + } + } + } } layer := store.layerPath(id) if !store.Exists(id) { diff --git a/server/server.go b/server/server.go index 8a3bb404f3..0e57c3da42 100644 --- a/server/server.go +++ b/server/server.go @@ -348,17 +348,9 @@ func (srv *Server) CmdKill(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "pull", "[OPTIONS] NAME", "Download a new image from a remote location") - fl_bzip2 := cmd.Bool("j", false, "Bzip2 compression") - fl_gzip := cmd.Bool("z", false, "Gzip compression") if err := cmd.Parse(args); err != nil { return nil } - var compression image.Compression - if *fl_bzip2 { - compression = image.Bzip2 - } else if *fl_gzip { - compression = image.Gzip - } name := cmd.Arg(0) if name == "" { return errors.New("Not enough arguments") @@ -375,12 +367,13 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string u.Host = "s3.amazonaws.com" u.Path = path.Join("/docker.io/images", u.Path) } - fmt.Fprintf(stdout, "Downloading %s from %s...\n", name, u.String()) + fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) resp, err := http.Get(u.String()) if err != nil { return err } - img, err := srv.images.Import(name, resp.Body, stdout, nil, compression) + fmt.Fprintf(stdout, "Unpacking to %s\n", name) + img, err := srv.images.Import(name, resp.Body, nil) if err != nil { return err } @@ -390,22 +383,14 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string func (srv *Server) CmdPut(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "put", "[OPTIONS] NAME", "Import a new image from a local archive.") - fl_bzip2 := cmd.Bool("j", false, "Bzip2 compression") - fl_gzip := cmd.Bool("z", false, "Gzip compression") if err := cmd.Parse(args); err != nil { return nil } - var compression image.Compression - if *fl_bzip2 { - compression = image.Bzip2 - } else if *fl_gzip { - compression = image.Gzip - } name := cmd.Arg(0) if name == "" { return errors.New("Not enough arguments") } - img, err := srv.images.Import(name, stdin, stdout, nil, compression) + img, err := srv.images.Import(name, stdin, nil) if err != nil { return err } @@ -558,13 +543,13 @@ func (srv *Server) CmdCommit(stdin io.ReadCloser, stdout io.Writer, args ...stri } if container := srv.containers.Get(containerName); container != nil { // FIXME: freeze the container before copying it to avoid data corruption? - rwTar, err := docker.Tar(container.Filesystem.RWPath) + rwTar, err := image.Tar(container.Filesystem.RWPath, image.Uncompressed) if err != nil { return err } // Create a new image from the container's base layers + a new layer from container changes parentImg := srv.images.Find(container.GetUserData("image")) - img, err := srv.images.Import(imgName, rwTar, stdout, parentImg, image.Uncompressed) + img, err := srv.images.Import(imgName, rwTar, parentImg) if err != nil { return err } diff --git a/utils.go b/utils.go index 10c617d806..520073e3ab 100644 --- a/utils.go +++ b/utils.go @@ -17,25 +17,6 @@ func Trunc(s string, maxlen int) string { return s[:maxlen] } -// Tar generates a tar archive from a filesystem path, and returns it as a stream. -// Path must point to a directory. - -func Tar(path string) (io.Reader, error) { - cmd := exec.Command("tar", "-C", path, "-c", ".") - output, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - if err := cmd.Start(); err != nil { - return nil, err - } - // FIXME: errors will not be passed because we don't wait for the command. - // Instead, consumers will hit EOF right away. - // This can be fixed by waiting for the process to exit, or for the first write - // on stdout, whichever comes first. - return output, nil -} - // Figure out the absolute path of our own binary func SelfPath() string { path, err := exec.LookPath(os.Args[0]) From ac15003c05a844aa08e591ff7675ff6a406f188b Mon Sep 17 00:00:00 2001 From: Brian McCallister Date: Sat, 23 Feb 2013 13:59:06 -0700 Subject: [PATCH 08/35] ignore .vagrant --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7c99eb899d..495c502187 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.vagrant docker dockerd .*.swp From 797bb6e75b8f33fe44932bf90145cf069f342e44 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 25 Feb 2013 10:45:23 -0800 Subject: [PATCH 09/35] Network allocator --- network.go | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ network_test.go | 27 +++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/network.go b/network.go index 1e9a2edd5d..977db53c90 100644 --- a/network.go +++ b/network.go @@ -3,6 +3,7 @@ package docker import ( "bytes" "encoding/binary" + "errors" "fmt" "math/rand" "net" @@ -118,3 +119,65 @@ func allocateNetwork() (*NetworkInterface, error) { } return iface, nil } + +type NetworkAllocator struct { + iface string + queue chan (net.IP) +} + +func (alloc *NetworkAllocator) Acquire() (net.IP, error) { + select { + case ip := <-alloc.queue: + return ip, nil + default: + return net.IP{}, errors.New("No more IP addresses available") + } + return net.IP{}, nil +} + +func (alloc *NetworkAllocator) Release(ip net.IP) error { + select { + case alloc.queue <- ip: + return nil + default: + return errors.New("Too many IP addresses have been released") + } + return nil +} + +func (alloc *NetworkAllocator) PopulateFromNetwork(network *net.IPNet) error { + firstIP, _ := networkRange(network) + size, err := networkSize(network.Mask) + if err != nil { + return err + } + // The queue size should be the network size - 3 + // -1 for the network address, -1 for the broadcast address and + // -1 for the gateway address + alloc.queue = make(chan net.IP, size-3) + for i := int32(1); i < size-1; i++ { + ipNum, err := ipToInt(firstIP) + if err != nil { + return err + } + ip, err := intToIp(ipNum + int32(i)) + if err != nil { + return err + } + // Discard the network IP (that's the host IP address) + if ip.Equal(network.IP) { + continue + } + alloc.Release(ip) + } + return nil +} + +func (alloc *NetworkAllocator) PopulateFromInterface(iface string) error { + addr, err := getBridgeAddr(iface) + if err != nil { + return err + } + network := addr.(*net.IPNet) + return alloc.PopulateFromNetwork(network) +} diff --git a/network_test.go b/network_test.go index c1ea382783..c3de398681 100644 --- a/network_test.go +++ b/network_test.go @@ -99,3 +99,30 @@ func TestConversion(t *testing.T) { t.Error(conv.String()) } } + +func TestNetworkAllocator(t *testing.T) { + alloc := NetworkAllocator{} + _, n, _ := net.ParseCIDR("127.0.0.1/29") + alloc.PopulateFromNetwork(n) + var lastIP net.IP + for i := 0; i < 5; i++ { + ip, err := alloc.Acquire() + if err != nil { + t.Fatal(err) + } + lastIP = ip + } + ip, err := alloc.Acquire() + if err == nil { + t.Fatal("There shouldn't be any IP addresses at this point") + } + // Release 1 IP + alloc.Release(lastIP) + ip, err = alloc.Acquire() + if err != nil { + t.Fatal(err) + } + if !ip.Equal(lastIP) { + t.Fatal(ip.String()) + } +} From 2c7a2cbaf4f3b47786eee3726de61b67182a8d44 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 25 Feb 2013 12:27:29 -0800 Subject: [PATCH 10/35] Moved Jeff's install script to the repo --- install.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 install.sh diff --git a/install.sh b/install.sh new file mode 100644 index 0000000000..356ac0df2a --- /dev/null +++ b/install.sh @@ -0,0 +1,34 @@ +# This script is meant for quick & easy install via 'curl URL-OF-SCRIPPT | bash' +# Courtesy of Jeff Lindsay + +cd /tmp + +echo "Ensuring dependencies are installed..." +apt-get --yes install lxc wget 2>&1 > /dev/null + +echo "Downloading docker binary..." +wget -q https://dl.dropbox.com/u/20637798/docker.tar.gz 2>&1 > /dev/null +tar -xf docker.tar.gz 2>&1 > /dev/null + +echo "Installing into /usr/local/bin..." +mv docker/docker /usr/local/bin +mv dockerd/dockerd /usr/local/bin + +if [[ -f /etc/init/dockerd.conf ]] +then + echo "Upstart script already exists." +else + echo "Creating /etc/init/dockerd.conf..." + echo "exec /usr/local/bin/dockerd" > /etc/init/dockerd.conf +fi + +echo "Restarting dockerd..." +restart dockerd > /dev/null + +echo "Cleaning up..." +rmdir docker +rmdir dockerd +rm docker.tar.gz + +echo "Finished!" +echo From edf2e20e284a6a3bb28e178745d2069d63261235 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Mon, 25 Feb 2013 12:27:51 -0800 Subject: [PATCH 11/35] Updated dependencies in install.sh --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 356ac0df2a..af5c214d78 100644 --- a/install.sh +++ b/install.sh @@ -4,7 +4,7 @@ cd /tmp echo "Ensuring dependencies are installed..." -apt-get --yes install lxc wget 2>&1 > /dev/null +apt-get --yes install lxc wget bsdtar 2>&1 > /dev/null echo "Downloading docker binary..." wget -q https://dl.dropbox.com/u/20637798/docker.tar.gz 2>&1 > /dev/null From c08f5b2b8460f13f2094bae2a496bf308f7645bb Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Mon, 25 Feb 2013 14:06:22 -0800 Subject: [PATCH 12/35] Integrated the network allocator into Docker. A networking environment is assigned to each container upon Start and released whenever the container exits. --- container.go | 75 ++++++++++++++++++++++++++++++++-------------- docker.go | 22 +++++++++----- lxc_template.go | 2 +- network.go | 79 +++++++++++++++++++++++-------------------------- network_test.go | 10 +++---- 5 files changed, 110 insertions(+), 78 deletions(-) diff --git a/container.go b/container.go index 50ac5e715e..67c450282f 100644 --- a/container.go +++ b/container.go @@ -33,9 +33,12 @@ type Container struct { Config *Config Filesystem *Filesystem - Network *NetworkInterface State *State + network *NetworkInterface + networkAllocator *NetworkAllocator + NetworkConfig *NetworkConfig + SysInitPath string lxcConfigPath string cmd *exec.Cmd @@ -56,16 +59,23 @@ type Config struct { OpenStdin bool // Open stdin } -func createContainer(id string, root string, command string, args []string, layers []string, config *Config) (*Container, error) { +type NetworkConfig struct { + IpAddress string + IpPrefixLen int +} + +func createContainer(id string, root string, command string, args []string, layers []string, config *Config, netAllocator *NetworkAllocator) (*Container, error) { container := &Container{ - Id: id, - Root: root, - Created: time.Now(), - Path: command, - Args: args, - Config: config, - Filesystem: newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers), - State: newState(), + Id: id, + Root: root, + Created: time.Now(), + Path: command, + Args: args, + Config: config, + Filesystem: newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers), + State: newState(), + networkAllocator: netAllocator, + NetworkConfig: &NetworkConfig{}, SysInitPath: sysInitPath, lxcConfigPath: path.Join(root, "config.lxc"), @@ -88,27 +98,25 @@ func createContainer(id string, root string, command string, args []string, laye if err := container.Filesystem.createMountPoints(); err != nil { return nil, err } - var err error - if container.Network, err = allocateNetwork(); err != nil { - return nil, err - } if err := container.save(); err != nil { return nil, err } return container, nil } -func loadContainer(containerPath string) (*Container, error) { +func loadContainer(containerPath string, netAllocator *NetworkAllocator) (*Container, error) { data, err := ioutil.ReadFile(path.Join(containerPath, "config.json")) if err != nil { return nil, err } container := &Container{ - stdout: newWriteBroadcaster(), - stderr: newWriteBroadcaster(), - stdoutLog: new(bytes.Buffer), - stderrLog: new(bytes.Buffer), - lxcConfigPath: path.Join(containerPath, "config.lxc"), + stdout: newWriteBroadcaster(), + stderr: newWriteBroadcaster(), + stdoutLog: new(bytes.Buffer), + stderrLog: new(bytes.Buffer), + lxcConfigPath: path.Join(containerPath, "config.lxc"), + networkAllocator: netAllocator, + NetworkConfig: &NetworkConfig{}, } if err := json.Unmarshal(data, container); err != nil { return nil, err @@ -268,6 +276,9 @@ func (container *Container) Start() error { if err := container.Filesystem.EnsureMounted(); err != nil { return err } + if err := container.allocateNetwork(); err != nil { + return err + } if err := container.generateLXCConfig(); err != nil { return err } @@ -279,7 +290,7 @@ func (container *Container) Start() error { } // Networking - params = append(params, "-g", container.Network.Gateway.String()) + params = append(params, "-g", container.network.Gateway.String()) // User if container.Config.User != "" { @@ -356,12 +367,33 @@ func (container *Container) StderrLog() io.Reader { return strings.NewReader(container.stderrLog.String()) } +func (container *Container) allocateNetwork() error { + iface, err := container.networkAllocator.Allocate() + if err != nil { + return err + } + container.network = iface + container.NetworkConfig.IpAddress = iface.IPNet.IP.String() + container.NetworkConfig.IpPrefixLen, _ = iface.IPNet.Mask.Size() + return nil +} + +func (container *Container) releaseNetwork() error { + err := container.networkAllocator.Release(container.network) + container.network = nil + container.NetworkConfig = &NetworkConfig{} + return err +} + func (container *Container) monitor() { // Wait for the program to exit container.cmd.Wait() exitCode := container.cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus() // Cleanup + if err := container.releaseNetwork(); err != nil { + log.Printf("%v: Failed to release network: %v", container.Id, err) + } container.stdout.Close() container.stderr.Close() if err := container.Filesystem.Umount(); err != nil { @@ -429,7 +461,6 @@ func (container *Container) Restart() error { } func (container *Container) Wait() { - for container.State.Running { container.State.wait() } diff --git a/docker.go b/docker.go index abe2d3777d..f44eb04059 100644 --- a/docker.go +++ b/docker.go @@ -11,9 +11,10 @@ import ( ) type Docker struct { - root string - repository string - containers *list.List + root string + repository string + containers *list.List + networkAllocator *NetworkAllocator } func (docker *Docker) List() []*Container { @@ -51,7 +52,7 @@ func (docker *Docker) Create(id string, command string, args []string, layers [] return nil, fmt.Errorf("Container %v already exists", id) } root := path.Join(docker.repository, id) - container, err := createContainer(id, root, command, args, layers, config) + container, err := createContainer(id, root, command, args, layers, config, docker.networkAllocator) if err != nil { return nil, err } @@ -86,7 +87,7 @@ func (docker *Docker) restore() error { return err } for _, v := range dir { - container, err := loadContainer(path.Join(docker.repository, v.Name())) + container, err := loadContainer(path.Join(docker.repository, v.Name()), docker.networkAllocator) if err != nil { log.Printf("Failed to load container %v: %v", v.Name(), err) continue @@ -101,10 +102,15 @@ func New() (*Docker, error) { } func NewFromDirectory(root string) (*Docker, error) { + alloc, err := newNetworkAllocator(networkBridgeIface) + if err != nil { + return nil, err + } docker := &Docker{ - root: root, - repository: path.Join(root, "containers"), - containers: list.New(), + root: root, + repository: path.Join(root, "containers"), + containers: list.New(), + networkAllocator: alloc, } if err := os.MkdirAll(docker.repository, 0700); err != nil && !os.IsExist(err) { diff --git a/lxc_template.go b/lxc_template.go index 48e6dc732b..3e66885460 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -19,7 +19,7 @@ lxc.network.flags = up lxc.network.link = lxcbr0 lxc.network.name = eth0 lxc.network.mtu = 1500 -lxc.network.ipv4 = {{.Network.IpAddress}}/{{.Network.IpPrefixLen}} +lxc.network.ipv4 = {{.NetworkConfig.IpAddress}}/{{.NetworkConfig.IpPrefixLen}} # root filesystem {{$ROOTFS := .Filesystem.RootFS}} diff --git a/network.go b/network.go index 977db53c90..db4048feb5 100644 --- a/network.go +++ b/network.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "errors" "fmt" - "math/rand" "net" ) @@ -14,11 +13,12 @@ const ( ) type NetworkInterface struct { - IpAddress string - IpPrefixLen int - Gateway net.IP + IPNet net.IPNet + Gateway net.IP } +// IP utils + func networkRange(network *net.IPNet) (net.IP, net.IP) { netIP := network.IP.To4() firstIP := netIP.Mask(network.Mask) @@ -51,10 +51,11 @@ func intToIp(n int32) (net.IP, error) { } func networkSize(mask net.IPMask) (int32, error) { + m := net.IPv4Mask(0, 0, 0, 0) for i := 0; i < net.IPv4len; i++ { - mask[i] = ^mask[i] + m[i] = ^mask[i] } - buf := bytes.NewBuffer(mask) + buf := bytes.NewBuffer(m) var n int32 if err := binary.Read(buf, binary.BigEndian, &n); err != nil { return 0, err @@ -62,21 +63,7 @@ func networkSize(mask net.IPMask) (int32, error) { return n + 1, nil } -func allocateIPAddress(network *net.IPNet) (net.IP, error) { - ip, _ := networkRange(network) - netSize, err := networkSize(network.Mask) - if err != nil { - return net.IP{}, err - } - numIp, err := ipToInt(ip) - if err != nil { - return net.IP{}, err - } - numIp += rand.Int31n(netSize) - return intToIp(numIp) -} - -func getBridgeAddr(name string) (net.Addr, error) { +func getIfaceAddr(name string) (net.Addr, error) { iface, err := net.InterfaceByName(name) if err != nil { return nil, err @@ -101,31 +88,31 @@ func getBridgeAddr(name string) (net.Addr, error) { return addrs4[0], nil } -func allocateNetwork() (*NetworkInterface, error) { - bridgeAddr, err := getBridgeAddr(networkBridgeIface) +// Network allocator +func newNetworkAllocator(iface string) (*NetworkAllocator, error) { + addr, err := getIfaceAddr(iface) if err != nil { return nil, err } - bridge := bridgeAddr.(*net.IPNet) - ipPrefixLen, _ := bridge.Mask.Size() - ip, err := allocateIPAddress(bridge) - if err != nil { + network := addr.(*net.IPNet) + + alloc := &NetworkAllocator{ + iface: iface, + net: network, + } + if err := alloc.populateFromNetwork(network); err != nil { return nil, err } - iface := &NetworkInterface{ - IpAddress: ip.String(), - IpPrefixLen: ipPrefixLen, - Gateway: bridge.IP, - } - return iface, nil + return alloc, nil } type NetworkAllocator struct { iface string + net *net.IPNet queue chan (net.IP) } -func (alloc *NetworkAllocator) Acquire() (net.IP, error) { +func (alloc *NetworkAllocator) acquireIP() (net.IP, error) { select { case ip := <-alloc.queue: return ip, nil @@ -135,7 +122,7 @@ func (alloc *NetworkAllocator) Acquire() (net.IP, error) { return net.IP{}, nil } -func (alloc *NetworkAllocator) Release(ip net.IP) error { +func (alloc *NetworkAllocator) releaseIP(ip net.IP) error { select { case alloc.queue <- ip: return nil @@ -145,7 +132,7 @@ func (alloc *NetworkAllocator) Release(ip net.IP) error { return nil } -func (alloc *NetworkAllocator) PopulateFromNetwork(network *net.IPNet) error { +func (alloc *NetworkAllocator) populateFromNetwork(network *net.IPNet) error { firstIP, _ := networkRange(network) size, err := networkSize(network.Mask) if err != nil { @@ -168,16 +155,24 @@ func (alloc *NetworkAllocator) PopulateFromNetwork(network *net.IPNet) error { if ip.Equal(network.IP) { continue } - alloc.Release(ip) + alloc.releaseIP(ip) } return nil } -func (alloc *NetworkAllocator) PopulateFromInterface(iface string) error { - addr, err := getBridgeAddr(iface) +func (alloc *NetworkAllocator) Allocate() (*NetworkInterface, error) { + // ipPrefixLen, _ := alloc.net.Mask.Size() + ip, err := alloc.acquireIP() if err != nil { - return err + return nil, err } - network := addr.(*net.IPNet) - return alloc.PopulateFromNetwork(network) + iface := &NetworkInterface{ + IPNet: net.IPNet{ip, alloc.net.Mask}, + Gateway: alloc.net.IP, + } + return iface, nil +} + +func (alloc *NetworkAllocator) Release(iface *NetworkInterface) error { + return alloc.releaseIP(iface.IPNet.IP) } diff --git a/network_test.go b/network_test.go index c3de398681..d66e7e5393 100644 --- a/network_test.go +++ b/network_test.go @@ -103,22 +103,22 @@ func TestConversion(t *testing.T) { func TestNetworkAllocator(t *testing.T) { alloc := NetworkAllocator{} _, n, _ := net.ParseCIDR("127.0.0.1/29") - alloc.PopulateFromNetwork(n) + alloc.populateFromNetwork(n) var lastIP net.IP for i := 0; i < 5; i++ { - ip, err := alloc.Acquire() + ip, err := alloc.acquireIP() if err != nil { t.Fatal(err) } lastIP = ip } - ip, err := alloc.Acquire() + ip, err := alloc.acquireIP() if err == nil { t.Fatal("There shouldn't be any IP addresses at this point") } // Release 1 IP - alloc.Release(lastIP) - ip, err = alloc.Acquire() + alloc.releaseIP(lastIP) + ip, err = alloc.acquireIP() if err != nil { t.Fatal(err) } From 6d1054619d6d254ea3df0484cacb3c6778d58cd9 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 26 Feb 2013 09:46:29 -0800 Subject: [PATCH 13/35] Added tests for checksum computation in layer store --- image/layers_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 image/layers_test.go diff --git a/image/layers_test.go b/image/layers_test.go new file mode 100644 index 0000000000..a69edb1892 --- /dev/null +++ b/image/layers_test.go @@ -0,0 +1,48 @@ +package image + +import ( + "os" + "testing" + "io/ioutil" + "bytes" + "github.com/dotcloud/docker/future" + "github.com/dotcloud/docker/fake" +) + +func TestAddLayer(t *testing.T) { + tmp, err := ioutil.TempDir("", "docker-test-image") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + store, err := NewLayerStore(tmp) + if err != nil { + t.Fatal(err) + } + archive, err := fake.FakeTar() + if err != nil { + t.Fatal(err) + } + layer, err := store.AddLayer(archive) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(layer); err != nil { + t.Fatalf("Error testing for existence of layer: %s\n", err.Error()) + } +} + +func TestComputeId(t *testing.T) { + id1, err := future.ComputeId(bytes.NewBufferString("hello world\n")) + if err != nil { + t.Fatal(err) + } + id2, err := future.ComputeId(bytes.NewBufferString("foo bar\n")) + if err != nil { + t.Fatal(err) + } + if id1 == id2 { + t.Fatalf("Identical checksums for difference content (%s == %s)", id1, id2) + } +} + From 8fa07c0e060a25141204533a2a3823d1cb0fdf63 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 26 Feb 2013 10:03:06 -0800 Subject: [PATCH 14/35] New example: pybuilder --- examples/pybuilder | 73 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 examples/pybuilder diff --git a/examples/pybuilder b/examples/pybuilder new file mode 100755 index 0000000000..a9a2e8276f --- /dev/null +++ b/examples/pybuilder @@ -0,0 +1,73 @@ +#!/usr/bin/env docker -i + +# Uncomment to debug: +#set -x + +export NORAW=1 + +IMG=shykes/pybuilder:11d4f58638a72935 + +if [ $# -lt 3 ]; then + echo "Usage: $0 build|run USER/REPO REV" + echo "Example usage:" + echo "" + echo " REV=7d5f035432fe1453eea389b0f1b02a2a93c8009e" + echo " $0 build shykes/helloflask \$REV" + echo " $0 run shykes/helloflask \$REV" + echo "" + exit 1 +fi + +CMD=$1 + +FORCE=0 +if [ "$2" = "-f" ]; then + FORCE=1 + shift +fi + +REPO=$2 +REV=$3 + +BUILD_IMAGE=builds/github.com/$REPO/$REV + + +if [ "$CMD" = "build" ]; then + if [ ! -z "`images -q $BUILD_IMAGE`" ]; then + if [ "$FORCE" -ne 1 ]; then + echo "$BUILD_IMAGE already exists" + exit + fi + fi + + # Allocate a TTY to work around python's aggressive buffering of stdout + BUILD_JOB=`run -t $IMG /usr/local/bin/buildapp http://github.com/$REPO/archive/$REV.tar.gz` + + if [ -z "$BUILD_JOB" ]; then + echo "Build failed" + exit 1 + fi + + if attach $BUILD_JOB ; then + BUILD_STATUS=`docker ps -a | sed -E -n "s/^$BUILD_JOB.*Exit ([0-9]+) *$/\1/p"` + if [ -z "$BUILD_STATUS" -o "$BUILD_STATUS" != 0 ]; then + echo "Build failed" + exit 1 + fi + + else + echo "Build failed" + exit 1 + fi + + commit $BUILD_JOB $BUILD_IMAGE + + echo "Build saved at $BUILD_IMAGE" +elif [ "$CMD" = "run" ]; then + RUN_JOB=`run $BUILD_IMAGE /usr/local/bin/runapp` + if [ -z "$RUN_JOB" ]; then + echo "Run failed" + exit 1 + fi + attach $RUN_JOB +fi From ebaa50c4c96a1ba69e480fab5612b5d6a97da145 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 26 Feb 2013 11:43:54 -0800 Subject: [PATCH 15/35] docker wait: block until a container exits, and print its exit code --- container.go | 4 +++- server/server.go | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/container.go b/container.go index 57ec531737..7c1caf4d57 100644 --- a/container.go +++ b/container.go @@ -415,11 +415,13 @@ func (container *Container) Restart() error { return nil } -func (container *Container) Wait() { +// Wait blocks until the container stops running, then returns its exit code. +func (container *Container) Wait() int { for container.State.Running { container.State.wait() } + return container.State.ExitCode } func (container *Container) WaitTimeout(timeout time.Duration) error { diff --git a/server/server.go b/server/server.go index 0e57c3da42..ee2c997634 100644 --- a/server/server.go +++ b/server/server.go @@ -53,6 +53,7 @@ func (srv *Server) Help() string { {"diff", "Inspect changes on a container's filesystem"}, {"commit", "Save the state of a container"}, {"attach", "Attach to the standard inputs and outputs of a running container"}, + {"wait", "Block until a container exits, then print its exit code"}, {"info", "Display system-wide information"}, {"tar", "Stream the contents of a container as a tar archive"}, {"web", "Generate a web UI"}, @@ -63,6 +64,27 @@ func (srv *Server) Help() string { return help } +// 'docker wait': block until a container stops +func (srv *Server) CmdWait(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "wait", "[OPTIONS] NAME", "Block until a container stops, then print its exit code.") + if err := cmd.Parse(args); err != nil { + cmd.Usage() + return nil + } + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } + for _, name := range cmd.Args() { + if container := srv.containers.Get(name); container != nil { + fmt.Fprintln(stdout, container.Wait()) + } else { + return errors.New("No such container: " + name) + } + } + return nil +} + // 'docker info': display system-wide information. func (srv *Server) CmdInfo(stdin io.ReadCloser, stdout io.Writer, args ...string) error { fmt.Fprintf(stdout, "containers: %d\nversion: %s\nimages: %d\n", From 4004e86fa9ca87332c8f5fa0acbe29af86e06908 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 26 Feb 2013 14:47:20 -0800 Subject: [PATCH 16/35] Updated pybuilder example to use 'docker wait' --- examples/pybuilder | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pybuilder b/examples/pybuilder index a9a2e8276f..dfab8ab2dd 100755 --- a/examples/pybuilder +++ b/examples/pybuilder @@ -49,7 +49,7 @@ if [ "$CMD" = "build" ]; then fi if attach $BUILD_JOB ; then - BUILD_STATUS=`docker ps -a | sed -E -n "s/^$BUILD_JOB.*Exit ([0-9]+) *$/\1/p"` + BUILD_STATUS=`docker wait $BUILD_JOB` if [ -z "$BUILD_STATUS" -o "$BUILD_STATUS" != 0 ]; then echo "Build failed" exit 1 From aa12da6f5076738955b30470639d439a0cbe40f9 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 26 Feb 2013 17:26:46 -0800 Subject: [PATCH 17/35] go fmt --- client/client.go | 4 +- client/term.go | 172 +++++++++++++++++++++--------------------- docker/docker.go | 3 +- fake/fake.go | 16 ++-- filesystem.go | 2 +- future/future.go | 25 +++--- image/archive.go | 12 +-- image/archive_test.go | 4 +- image/image.go | 55 +++++++------- image/layers.go | 17 ++--- image/layers_test.go | 9 +-- mount_linux.go | 1 - 12 files changed, 152 insertions(+), 168 deletions(-) diff --git a/client/client.go b/client/client.go index 5a8aac3807..4c4ea1c5e3 100644 --- a/client/client.go +++ b/client/client.go @@ -1,8 +1,8 @@ package client import ( - "github.com/dotcloud/docker/rcli" "github.com/dotcloud/docker/future" + "github.com/dotcloud/docker/rcli" "io" "io/ioutil" "log" @@ -112,7 +112,7 @@ func InteractiveMode(scripts ...string) error { return err } io.WriteString(rcfile, "enable -n help\n") - os.Setenv("PATH", tmp + ":" + os.Getenv("PATH")) + os.Setenv("PATH", tmp+":"+os.Getenv("PATH")) os.Setenv("PS1", "\\h docker> ") shell := exec.Command("/bin/bash", append([]string{"--rcfile", rcfile.Name()}, scripts...)...) shell.Stdin = os.Stdin diff --git a/client/term.go b/client/term.go index ed52be96b4..a988d0d796 100644 --- a/client/term.go +++ b/client/term.go @@ -15,7 +15,6 @@ type Termios struct { Ospeed uintptr } - const ( // Input flags inpck = 0x010 @@ -35,113 +34,110 @@ const ( ) const ( - HUPCL = 0x4000 - ICANON = 0x100 - ICRNL = 0x100 - IEXTEN = 0x400 - BRKINT = 0x2 - CFLUSH = 0xf - CLOCAL = 0x8000 - CREAD = 0x800 - CS5 = 0x0 - CS6 = 0x100 - CS7 = 0x200 - CS8 = 0x300 - CSIZE = 0x300 - CSTART = 0x11 - CSTATUS = 0x14 - CSTOP = 0x13 - CSTOPB = 0x400 - CSUSP = 0x1a - IGNBRK = 0x1 - IGNCR = 0x80 - IGNPAR = 0x4 - IMAXBEL = 0x2000 - INLCR = 0x40 - INPCK = 0x10 - ISIG = 0x80 - ISTRIP = 0x20 - IUTF8 = 0x4000 - IXANY = 0x800 - IXOFF = 0x400 - IXON = 0x200 - NOFLSH = 0x80000000 - OCRNL = 0x10 - OFDEL = 0x20000 - OFILL = 0x80 - ONLCR = 0x2 - ONLRET = 0x40 - ONOCR = 0x20 - ONOEOT = 0x8 - OPOST = 0x1 -RENB = 0x1000 - PARMRK = 0x8 - PARODD = 0x2000 + HUPCL = 0x4000 + ICANON = 0x100 + ICRNL = 0x100 + IEXTEN = 0x400 + BRKINT = 0x2 + CFLUSH = 0xf + CLOCAL = 0x8000 + CREAD = 0x800 + CS5 = 0x0 + CS6 = 0x100 + CS7 = 0x200 + CS8 = 0x300 + CSIZE = 0x300 + CSTART = 0x11 + CSTATUS = 0x14 + CSTOP = 0x13 + CSTOPB = 0x400 + CSUSP = 0x1a + IGNBRK = 0x1 + IGNCR = 0x80 + IGNPAR = 0x4 + IMAXBEL = 0x2000 + INLCR = 0x40 + INPCK = 0x10 + ISIG = 0x80 + ISTRIP = 0x20 + IUTF8 = 0x4000 + IXANY = 0x800 + IXOFF = 0x400 + IXON = 0x200 + NOFLSH = 0x80000000 + OCRNL = 0x10 + OFDEL = 0x20000 + OFILL = 0x80 + ONLCR = 0x2 + ONLRET = 0x40 + ONOCR = 0x20 + ONOEOT = 0x8 + OPOST = 0x1 + RENB = 0x1000 + PARMRK = 0x8 + PARODD = 0x2000 - TOSTOP = 0x400000 - VDISCARD = 0xf - VDSUSP = 0xb - VEOF = 0x0 - VEOL = 0x1 - VEOL2 = 0x2 - VERASE = 0x3 - VINTR = 0x8 - VKILL = 0x5 - VLNEXT = 0xe - VMIN = 0x10 - VQUIT = 0x9 - VREPRINT = 0x6 - VSTART = 0xc - VSTATUS = 0x12 - VSTOP = 0xd - VSUSP = 0xa - VT0 = 0x0 - VT1 = 0x10000 - VTDLY = 0x10000 - VTIME = 0x11 - ECHO = 0x00000008 + TOSTOP = 0x400000 + VDISCARD = 0xf + VDSUSP = 0xb + VEOF = 0x0 + VEOL = 0x1 + VEOL2 = 0x2 + VERASE = 0x3 + VINTR = 0x8 + VKILL = 0x5 + VLNEXT = 0xe + VMIN = 0x10 + VQUIT = 0x9 + VREPRINT = 0x6 + VSTART = 0xc + VSTATUS = 0x12 + VSTOP = 0xd + VSUSP = 0xa + VT0 = 0x0 + VT1 = 0x10000 + VTDLY = 0x10000 + VTIME = 0x11 + ECHO = 0x00000008 - PENDIN = 0x20000000 + PENDIN = 0x20000000 ) type State struct { - termios Termios + termios Termios } // IsTerminal returns true if the given file descriptor is a terminal. func IsTerminal(fd int) bool { - var termios Termios - _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&termios)), 0, 0, 0) - return err == 0 + var termios Termios + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&termios)), 0, 0, 0) + return err == 0 } // MakeRaw put the terminal connected to the given file descriptor into raw // mode and returns the previous state of the terminal so that it can be // restored. func MakeRaw(fd int) (*State, error) { - var oldState State - if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 { - return nil, err - } + var oldState State + if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(getTermios), uintptr(unsafe.Pointer(&oldState.termios)), 0, 0, 0); err != 0 { + return nil, err + } - newState := oldState.termios - newState.Iflag &^= ISTRIP | INLCR | IGNCR | IXON | IXOFF - newState.Iflag |= ICRNL - newState.Oflag |= ONLCR - newState.Lflag &^= ECHO | ICANON | ISIG - if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 { - return nil, err - } + newState := oldState.termios + newState.Iflag &^= ISTRIP | INLCR | IGNCR | IXON | IXOFF + newState.Iflag |= ICRNL + newState.Oflag |= ONLCR + newState.Lflag &^= ECHO | ICANON | ISIG + if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&newState)), 0, 0, 0); err != 0 { + return nil, err + } - return &oldState, nil + return &oldState, nil } - // Restore restores the terminal connected to the given file descriptor to a // previous state. func Restore(fd int, state *State) error { - _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0) - return err + _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0) + return err } - - diff --git a/docker/docker.go b/docker/docker.go index efc93620a4..fa9011defa 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -2,10 +2,10 @@ package main import ( "flag" + "github.com/dotcloud/docker/client" "log" "os" "path" - "github.com/dotcloud/docker/client" ) func main() { @@ -27,4 +27,3 @@ func main() { } } } - diff --git a/fake/fake.go b/fake/fake.go index c1e694515b..8edbd88e6b 100644 --- a/fake/fake.go +++ b/fake/fake.go @@ -1,20 +1,19 @@ package fake import ( - "bytes" - "math/rand" - "io" "archive/tar" - "os/exec" + "bytes" "github.com/kr/pty" + "io" + "math/rand" + "os/exec" ) - func FakeTar() (io.Reader, error) { content := []byte("Hello world!\n") buf := new(bytes.Buffer) tw := tar.NewWriter(buf) - for _, name := range []string {"hello", "etc/postgres/postgres.conf", "etc/passwd", "var/log/postgres/postgres.conf"} { + for _, name := range []string{"hello", "etc/postgres/postgres.conf", "etc/passwd", "var/log/postgres/postgres.conf"} { hdr := new(tar.Header) hdr.Size = int64(len(content)) hdr.Name = name @@ -27,7 +26,6 @@ func FakeTar() (io.Reader, error) { return buf, nil } - func WriteFakeTar(dst io.Writer) error { if data, err := FakeTar(); err != nil { return err @@ -37,7 +35,6 @@ func WriteFakeTar(dst io.Writer) error { return nil } - func RandomBytesChanged() uint { return uint(rand.Int31n(24 * 1024 * 1024)) } @@ -54,7 +51,6 @@ func ContainerRunning() bool { return false } - func StartCommand(cmd *exec.Cmd, interactive bool) (io.WriteCloser, io.ReadCloser, error) { if interactive { term, err := pty.Start(cmd) @@ -76,5 +72,3 @@ func StartCommand(cmd *exec.Cmd, interactive bool) (io.WriteCloser, io.ReadClose } return stdin, stdout, nil } - - diff --git a/filesystem.go b/filesystem.go index eadc6f69be..26686b3a59 100644 --- a/filesystem.go +++ b/filesystem.go @@ -3,6 +3,7 @@ package docker import ( "errors" "fmt" + "github.com/dotcloud/docker/image" "io" "io/ioutil" "os" @@ -10,7 +11,6 @@ import ( "strings" "syscall" "time" - "github.com/dotcloud/docker/image" ) type Filesystem struct { diff --git a/future/future.go b/future/future.go index b1427e1007..75caab3d19 100644 --- a/future/future.go +++ b/future/future.go @@ -1,12 +1,12 @@ package future import ( - "crypto/sha256" - "io" - "fmt" - "time" "bytes" + "crypto/sha256" + "fmt" + "io" "math/rand" + "time" ) func Seed() { @@ -30,18 +30,18 @@ func HumanDuration(d time.Duration) string { return "About a minute" } else if minutes < 60 { return fmt.Sprintf("%d minutes", minutes) - } else if hours := int(d.Hours()); hours == 1{ + } else if hours := int(d.Hours()); hours == 1 { return "About an hour" } else if hours < 48 { return fmt.Sprintf("%d hours", hours) - } else if hours < 24 * 7 * 2 { - return fmt.Sprintf("%d days", hours / 24) - } else if hours < 24 * 30 * 3 { - return fmt.Sprintf("%d weeks", hours / 24 / 7) - } else if hours < 24 * 365 * 2 { - return fmt.Sprintf("%d months", hours / 24 / 30) + } else if hours < 24*7*2 { + return fmt.Sprintf("%d days", hours/24) + } else if hours < 24*30*3 { + return fmt.Sprintf("%d weeks", hours/24/7) + } else if hours < 24*365*2 { + return fmt.Sprintf("%d months", hours/24/30) } - return fmt.Sprintf("%d years", d.Hours() / 24 / 365) + return fmt.Sprintf("%d years", d.Hours()/24/365) } func randomBytes() io.Reader { @@ -83,4 +83,3 @@ func Pv(src io.Reader, info io.Writer) io.Reader { }() return r } - diff --git a/image/archive.go b/image/archive.go index bc8edb4bca..501d549cc8 100644 --- a/image/archive.go +++ b/image/archive.go @@ -1,30 +1,32 @@ package image import ( + "errors" "io" "io/ioutil" "os/exec" - "errors" ) type Compression uint32 const ( - Uncompressed Compression = iota + Uncompressed Compression = iota Bzip2 Gzip ) func (compression *Compression) Flag() string { switch *compression { - case Bzip2: return "j" - case Gzip: return "z" + case Bzip2: + return "j" + case Gzip: + return "z" } return "" } func Tar(path string, compression Compression) (io.Reader, error) { - cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-c" + compression.Flag(), ".") + cmd := exec.Command("bsdtar", "-f", "-", "-C", path, "-c"+compression.Flag(), ".") return CmdStream(cmd) } diff --git a/image/archive_test.go b/image/archive_test.go index 0c19e605fe..4271849797 100644 --- a/image/archive_test.go +++ b/image/archive_test.go @@ -1,10 +1,10 @@ package image import ( - "testing" + "io/ioutil" "os" "os/exec" - "io/ioutil" + "testing" ) func TestCmdStreamBad(t *testing.T) { diff --git a/image/image.go b/image/image.go index d819237a7e..ebc425db09 100644 --- a/image/image.go +++ b/image/image.go @@ -1,27 +1,25 @@ package image import ( + "encoding/json" + "errors" + "github.com/dotcloud/docker/future" "io" "io/ioutil" - "encoding/json" - "time" + "os" "path" "path/filepath" - "errors" "sort" - "os" - "github.com/dotcloud/docker/future" "strings" + "time" ) - type Store struct { *Index - Root string - Layers *LayerStore + Root string + Layers *LayerStore } - func New(root string) (*Store, error) { abspath, err := filepath.Abs(root) if err != nil { @@ -38,8 +36,8 @@ func New(root string) (*Store, error) { return nil, err } return &Store{ - Root: abspath, - Index: NewIndex(path.Join(root, "index.json")), + Root: abspath, + Index: NewIndex(path.Join(root, "index.json")), Layers: layers, }, nil } @@ -73,20 +71,19 @@ func (store *Store) Create(name string, source string, layers ...string) (*Image return image, nil } - // Index type Index struct { - Path string - ByName map[string]*History - ById map[string]*Image + Path string + ByName map[string]*History + ById map[string]*Image } func NewIndex(path string) *Index { return &Index{ - Path: path, + Path: path, ByName: make(map[string]*History), - ById: make(map[string]*Image), + ById: make(map[string]*Image), } } @@ -216,7 +213,7 @@ func (index *Index) Names() []string { if err := index.load(); err != nil { return []string{} } - var names[]string + var names []string for name := range index.ByName { names = append(names, name) } @@ -279,23 +276,23 @@ func (history *History) Add(image *Image) { func (history *History) Del(id string) { for idx, image := range *history { if image.Id == id { - *history = append((*history)[:idx], (*history)[idx + 1:]...) + *history = append((*history)[:idx], (*history)[idx+1:]...) } } } type Image struct { - Id string // Globally unique identifier - Layers []string // Absolute paths - Created time.Time - Parent string + Id string // Globally unique identifier + Layers []string // Absolute paths + Created time.Time + Parent string } func (image *Image) IdParts() (string, string) { if len(image.Id) < 8 { return "", image.Id } - hash := image.Id[len(image.Id)-8:len(image.Id)] + hash := image.Id[len(image.Id)-8 : len(image.Id)] name := image.Id[:len(image.Id)-9] return name, hash } @@ -316,7 +313,7 @@ func generateImageId(name string, layers []string) (string, error) { for _, layer := range layers { ids += path.Base(layer) } - if h, err := future.ComputeId(strings.NewReader(ids)); err != nil { + if h, err := future.ComputeId(strings.NewReader(ids)); err != nil { return "", err } else { hash = h @@ -331,9 +328,9 @@ func NewImage(name string, layers []string, parent string) (*Image, error) { return nil, err } return &Image{ - Id: id, - Layers: layers, - Created: time.Now(), - Parent: parent, + Id: id, + Layers: layers, + Created: time.Now(), + Parent: parent, }, nil } diff --git a/image/layers.go b/image/layers.go index 61782cd88c..7a24b7bd57 100644 --- a/image/layers.go +++ b/image/layers.go @@ -2,16 +2,16 @@ package image import ( "errors" - "path" - "path/filepath" + "github.com/dotcloud/docker/future" "io" "io/ioutil" "os" - "github.com/dotcloud/docker/future" + "path" + "path/filepath" ) type LayerStore struct { - Root string + Root string } func NewLayerStore(root string) (*LayerStore, error) { @@ -66,10 +66,9 @@ func (store *LayerStore) Init() error { return os.Mkdir(store.Root, 0700) } - func (store *LayerStore) Mktemp() (string, error) { tmpName := future.RandomId() - tmpPath := path.Join(store.Root, "tmp-" + tmpName) + tmpPath := path.Join(store.Root, "tmp-"+tmpName) if err := os.Mkdir(tmpPath, 0700); err != nil { return "", err } @@ -80,7 +79,6 @@ func (store *LayerStore) layerPath(id string) string { return path.Join(store.Root, id) } - func (store *LayerStore) AddLayer(archive io.Reader) (string, error) { errors := make(chan error) // Untar @@ -109,9 +107,10 @@ func (store *LayerStore) AddLayer(archive io.Reader) (string, error) { return "", err } // Wait for goroutines - for i:=0; i<2; i+=1 { + for i := 0; i < 2; i += 1 { select { - case err := <-errors: { + case err := <-errors: + { if err != nil { return "", err } diff --git a/image/layers_test.go b/image/layers_test.go index a69edb1892..eec59ab3d0 100644 --- a/image/layers_test.go +++ b/image/layers_test.go @@ -1,12 +1,12 @@ package image import ( + "bytes" + "github.com/dotcloud/docker/fake" + "github.com/dotcloud/docker/future" + "io/ioutil" "os" "testing" - "io/ioutil" - "bytes" - "github.com/dotcloud/docker/future" - "github.com/dotcloud/docker/fake" ) func TestAddLayer(t *testing.T) { @@ -45,4 +45,3 @@ func TestComputeId(t *testing.T) { t.Fatalf("Identical checksums for difference content (%s == %s)", id1, id2) } } - diff --git a/mount_linux.go b/mount_linux.go index a5a24e8480..0efb253003 100644 --- a/mount_linux.go +++ b/mount_linux.go @@ -2,7 +2,6 @@ package docker import "syscall" - func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { return syscall.Mount(source, target, fstype, flags, data) } From 003ec21d360b08d50b61834795692542fff295e2 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 26 Feb 2013 17:28:24 -0800 Subject: [PATCH 18/35] If curl is installed, 'docker pull' will use it to download images with a pretty progress bar. Otherwise it will fallback to regular http.Get() --- README.md | 2 +- future/future.go | 17 +++++++++++++++++ server/server.go | 12 +++++++++--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cf3f74dc61..aea3d10e9d 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ Step by step host setup 3. Type the following commands: apt-get update - apt-get install lxc wget bsdtar + apt-get install lxc wget bsdtar curl 4. Download the latest version of the [docker binaries](https://dl.dropbox.com/u/20637798/docker.tar.gz) (`wget https://dl.dropbox.com/u/20637798/docker.tar.gz`) (warning: this may not be the most up-to-date build) 5. Extract the contents of the tar file `tar -xf docker.tar.gz` diff --git a/future/future.go b/future/future.go index 75caab3d19..33f1f8925c 100644 --- a/future/future.go +++ b/future/future.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "math/rand" + "os/exec" "time" ) @@ -83,3 +84,19 @@ func Pv(src io.Reader, info io.Writer) io.Reader { }() return r } + +// Curl makes an http request by executing the unix command 'curl', and returns +// the body of the response. If `stderr` is not nil, a progress bar will be +// written to it. +func Curl(url string, stderr io.Writer) (io.Reader, error) { + curl := exec.Command("curl", "-#", "-L", url) + output, err := curl.StdoutPipe() + if err != nil { + return nil, err + } + curl.Stderr = stderr + if err := curl.Start(); err != nil { + return nil, err + } + return output, nil +} diff --git a/server/server.go b/server/server.go index ee2c997634..2ef42f8b16 100644 --- a/server/server.go +++ b/server/server.go @@ -390,12 +390,18 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string u.Path = path.Join("/docker.io/images", u.Path) } fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) - resp, err := http.Get(u.String()) + // Download with curl (pretty progress bar) + // If curl is not available, fallback to http.Get() + archive, err := future.Curl(u.String(), stdout) if err != nil { - return err + if resp, err := http.Get(u.String()); err != nil { + return err + } else { + archive = resp.Body + } } fmt.Fprintf(stdout, "Unpacking to %s\n", name) - img, err := srv.images.Import(name, resp.Body, nil) + img, err := srv.images.Import(name, archive, nil) if err != nil { return err } From 278aa5f045ad6dd8718933b4736c29567a17717a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 26 Feb 2013 17:29:11 -0800 Subject: [PATCH 19/35] Changed .gitignore to not match docker/*.go --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 495c502187..fc4ca5da3b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .vagrant -docker -dockerd +docker/docker +dockerd/dockerd .*.swp a.out From ed85cb6508854f6d12d43e8a88a8c728bc633c3c Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Tue, 26 Feb 2013 18:17:51 -0800 Subject: [PATCH 20/35] Update puppet/modules/docker/manifests/init.pp Add required bsdtar dependency --- puppet/modules/docker/manifests/init.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/puppet/modules/docker/manifests/init.pp b/puppet/modules/docker/manifests/init.pp index 7c79892d4e..2923b302fc 100644 --- a/puppet/modules/docker/manifests/init.pp +++ b/puppet/modules/docker/manifests/init.pp @@ -8,7 +8,7 @@ class docker { Package { ensure => "installed" } - package { ["lxc", "debootstrap", "wget"]: } + package { ["lxc", "debootstrap", "wget", "bsdtar"]: } exec { "debootstrap" : require => Package["debootstrap"], From 799ffa176399877e610ff2049dbf84610037a2be Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 28 Feb 2013 11:50:02 -0800 Subject: [PATCH 21/35] Network: Port mapping support. Implemented a port allocator and a port mapper that is able to forward TCP ports from the host to the container. --- network.go | 290 ++++++++++++++++++++++++++++++++++++++---------- network_test.go | 18 +-- 2 files changed, 244 insertions(+), 64 deletions(-) diff --git a/network.go b/network.go index db4048feb5..1b6395b0b1 100644 --- a/network.go +++ b/network.go @@ -5,20 +5,20 @@ import ( "encoding/binary" "errors" "fmt" + "log" "net" + "os/exec" + "strconv" + "strings" ) const ( networkBridgeIface = "lxcbr0" + portRangeStart = 49153 + portRangeEnd = 65535 ) -type NetworkInterface struct { - IPNet net.IPNet - Gateway net.IP -} - -// IP utils - +// Calculates the first and last IP addresses in an IPNet func networkRange(network *net.IPNet) (net.IP, net.IP) { netIP := network.IP.To4() firstIP := netIP.Mask(network.Mask) @@ -29,6 +29,7 @@ func networkRange(network *net.IPNet) (net.IP, net.IP) { return firstIP, lastIP } +// Converts a 4 bytes IP into a 32 bit integer func ipToInt(ip net.IP) (int32, error) { buf := bytes.NewBuffer(ip.To4()) var n int32 @@ -38,6 +39,7 @@ func ipToInt(ip net.IP) (int32, error) { return n, nil } +// Converts 32 bit integer into a 4 bytes IP address func intToIp(n int32) (net.IP, error) { var buf bytes.Buffer if err := binary.Write(&buf, binary.BigEndian, &n); err != nil { @@ -50,6 +52,7 @@ func intToIp(n int32) (net.IP, error) { return ip, nil } +// Given a netmask, calculates the number of available hosts func networkSize(mask net.IPMask) (int32, error) { m := net.IPv4Mask(0, 0, 0, 0) for i := 0; i < net.IPv4len; i++ { @@ -63,6 +66,15 @@ func networkSize(mask net.IPMask) (int32, error) { return n + 1, nil } +// Wrapper around the iptables command +func iptables(args ...string) error { + if err := exec.Command("/sbin/iptables", args...).Run(); err != nil { + return fmt.Errorf("iptables failed: iptables %v", strings.Join(args, " ")) + } + return nil +} + +// Return the IPv4 address of a network interface func getIfaceAddr(name string) (net.Addr, error) { iface, err := net.InterfaceByName(name) if err != nil { @@ -81,60 +93,122 @@ func getIfaceAddr(name string) (net.Addr, error) { } switch { case len(addrs4) == 0: - return nil, fmt.Errorf("Bridge %v has no IP addresses", name) + return nil, fmt.Errorf("Interface %v has no IP addresses", name) case len(addrs4) > 1: - return nil, fmt.Errorf("Bridge %v has more than 1 IPv4 address", name) + return nil, fmt.Errorf("Interface %v has more than 1 IPv4 address", name) } return addrs4[0], nil } -// Network allocator -func newNetworkAllocator(iface string) (*NetworkAllocator, error) { - addr, err := getIfaceAddr(iface) - if err != nil { - return nil, err - } - network := addr.(*net.IPNet) - - alloc := &NetworkAllocator{ - iface: iface, - net: network, - } - if err := alloc.populateFromNetwork(network); err != nil { - return nil, err - } - return alloc, nil +// Port mapper takes care of mapping external ports to containers by setting +// up iptables rules. +// It keeps track of all mappings and is able to unmap at will +type PortMapper struct { + mapping map[int]net.TCPAddr } -type NetworkAllocator struct { - iface string - net *net.IPNet - queue chan (net.IP) +func (mapper *PortMapper) cleanup() error { + // Ignore errors - This could mean the chains were never set up + iptables("-t", "nat", "-D", "PREROUTING", "-j", "DOCKER") + iptables("-t", "nat", "-F", "DOCKER") + iptables("-t", "nat", "-X", "DOCKER") + mapper.mapping = make(map[int]net.TCPAddr) + return nil } -func (alloc *NetworkAllocator) acquireIP() (net.IP, error) { - select { - case ip := <-alloc.queue: - return ip, nil - default: - return net.IP{}, errors.New("No more IP addresses available") +func (mapper *PortMapper) setup() error { + if err := iptables("-t", "nat", "-N", "DOCKER"); err != nil { + return errors.New("Unable to setup port networking: Failed to create DOCKER chain") } - return net.IP{}, nil -} - -func (alloc *NetworkAllocator) releaseIP(ip net.IP) error { - select { - case alloc.queue <- ip: - return nil - default: - return errors.New("Too many IP addresses have been released") + if err := iptables("-t", "nat", "-A", "PREROUTING", "-j", "DOCKER"); err != nil { + return errors.New("Unable to setup port networking: Failed to inject docker in PREROUTING chain") } return nil } -func (alloc *NetworkAllocator) populateFromNetwork(network *net.IPNet) error { - firstIP, _ := networkRange(network) - size, err := networkSize(network.Mask) +func (mapper *PortMapper) iptablesForward(rule string, port int, dest net.TCPAddr) error { + return iptables("-t", "nat", rule, "DOCKER", "-p", "tcp", "--dport", strconv.Itoa(port), + "-j", "DNAT", "--to-destination", net.JoinHostPort(dest.IP.String(), strconv.Itoa(dest.Port))) +} + +func (mapper *PortMapper) Map(port int, dest net.TCPAddr) error { + if err := mapper.iptablesForward("-A", port, dest); err != nil { + return err + } + mapper.mapping[port] = dest + return nil +} + +func (mapper *PortMapper) Unmap(port int) error { + dest, ok := mapper.mapping[port] + if !ok { + return errors.New("Port is not mapped") + } + if err := mapper.iptablesForward("-D", port, dest); err != nil { + return err + } + delete(mapper.mapping, port) + return nil +} + +func newPortMapper() (*PortMapper, error) { + mapper := &PortMapper{} + if err := mapper.cleanup(); err != nil { + return nil, err + } + if err := mapper.setup(); err != nil { + return nil, err + } + return mapper, nil +} + +// Port allocator: Atomatically allocate and release networking ports +type PortAllocator struct { + ports chan (int) +} + +func (alloc *PortAllocator) populate(start, end int) { + alloc.ports = make(chan int, end-start) + for port := start; port < end; port++ { + alloc.ports <- port + } +} + +func (alloc *PortAllocator) Acquire() (int, error) { + select { + case port := <-alloc.ports: + return port, nil + default: + return -1, errors.New("No more ports available") + } + return -1, nil +} + +func (alloc *PortAllocator) Release(port int) error { + select { + case alloc.ports <- port: + return nil + default: + return errors.New("Too many ports have been released") + } + return nil +} + +func newPortAllocator(start, end int) (*PortAllocator, error) { + allocator := &PortAllocator{} + allocator.populate(start, end) + return allocator, nil +} + +// IP allocator: Atomatically allocate and release networking ports +type IPAllocator struct { + network *net.IPNet + queue chan (net.IP) +} + +func (alloc *IPAllocator) populate() error { + firstIP, _ := networkRange(alloc.network) + size, err := networkSize(alloc.network.Mask) if err != nil { return err } @@ -152,27 +226,131 @@ func (alloc *NetworkAllocator) populateFromNetwork(network *net.IPNet) error { return err } // Discard the network IP (that's the host IP address) - if ip.Equal(network.IP) { + if ip.Equal(alloc.network.IP) { continue } - alloc.releaseIP(ip) + alloc.queue <- ip } return nil } -func (alloc *NetworkAllocator) Allocate() (*NetworkInterface, error) { - // ipPrefixLen, _ := alloc.net.Mask.Size() - ip, err := alloc.acquireIP() +func (alloc *IPAllocator) Acquire() (net.IP, error) { + select { + case ip := <-alloc.queue: + return ip, nil + default: + return net.IP{}, errors.New("No more IP addresses available") + } + return net.IP{}, nil +} + +func (alloc *IPAllocator) Release(ip net.IP) error { + select { + case alloc.queue <- ip: + return nil + default: + return errors.New("Too many IP addresses have been released") + } + return nil +} + +func newIPAllocator(network *net.IPNet) (*IPAllocator, error) { + alloc := &IPAllocator{ + network: network, + } + if err := alloc.populate(); err != nil { + return nil, err + } + return alloc, nil +} + +// Network interface represents the networking stack of a container +type NetworkInterface struct { + IPNet net.IPNet + Gateway net.IP + + manager *NetworkManager + extPorts []int +} + +// Allocate an external TCP port and map it to the interface +func (iface *NetworkInterface) AllocatePort(port int) (int, error) { + extPort, err := iface.manager.portAllocator.Acquire() + if err != nil { + return -1, err + } + if err := iface.manager.portMapper.Map(extPort, net.TCPAddr{iface.IPNet.IP, port}); err != nil { + iface.manager.portAllocator.Release(extPort) + return -1, err + } + iface.extPorts = append(iface.extPorts, extPort) + return extPort, nil +} + +// Release: Network cleanup - release all resources +func (iface *NetworkInterface) Release() error { + for _, port := range iface.extPorts { + if err := iface.manager.portMapper.Unmap(port); err != nil { + log.Printf("Unable to unmap port %v: %v", port, err) + } + if err := iface.manager.portAllocator.Release(port); err != nil { + log.Printf("Unable to release port %v: %v", port, err) + } + + } + return iface.manager.ipAllocator.Release(iface.IPNet.IP) +} + +// Network Manager manages a set of network interfaces +// Only *one* manager per host machine should be used +type NetworkManager struct { + bridgeIface string + bridgeNetwork *net.IPNet + + ipAllocator *IPAllocator + portAllocator *PortAllocator + portMapper *PortMapper +} + +// Allocate a network interface +func (manager *NetworkManager) Allocate() (*NetworkInterface, error) { + ip, err := manager.ipAllocator.Acquire() if err != nil { return nil, err } iface := &NetworkInterface{ - IPNet: net.IPNet{ip, alloc.net.Mask}, - Gateway: alloc.net.IP, + IPNet: net.IPNet{ip, manager.bridgeNetwork.Mask}, + Gateway: manager.bridgeNetwork.IP, + manager: manager, } return iface, nil } -func (alloc *NetworkAllocator) Release(iface *NetworkInterface) error { - return alloc.releaseIP(iface.IPNet.IP) +func newNetworkManager(bridgeIface string) (*NetworkManager, error) { + addr, err := getIfaceAddr(bridgeIface) + if err != nil { + return nil, err + } + network := addr.(*net.IPNet) + + ipAllocator, err := newIPAllocator(network) + if err != nil { + return nil, err + } + + portAllocator, err := newPortAllocator(portRangeStart, portRangeEnd) + if err != nil { + return nil, err + } + + portMapper, err := newPortMapper() + + manager := &NetworkManager{ + bridgeIface: bridgeIface, + bridgeNetwork: network, + ipAllocator: ipAllocator, + portAllocator: portAllocator, + portMapper: portMapper, + } + return manager, nil } diff --git a/network_test.go b/network_test.go index d66e7e5393..c456b54838 100644 --- a/network_test.go +++ b/network_test.go @@ -100,25 +100,27 @@ func TestConversion(t *testing.T) { } } -func TestNetworkAllocator(t *testing.T) { - alloc := NetworkAllocator{} - _, n, _ := net.ParseCIDR("127.0.0.1/29") - alloc.populateFromNetwork(n) +func TestIPAllocator(t *testing.T) { + gwIP, n, _ := net.ParseCIDR("127.0.0.1/29") + alloc, err := newIPAllocator(&net.IPNet{gwIP, n.Mask}) + if err != nil { + t.Fatal(err) + } var lastIP net.IP for i := 0; i < 5; i++ { - ip, err := alloc.acquireIP() + ip, err := alloc.Acquire() if err != nil { t.Fatal(err) } lastIP = ip } - ip, err := alloc.acquireIP() + ip, err := alloc.Acquire() if err == nil { t.Fatal("There shouldn't be any IP addresses at this point") } // Release 1 IP - alloc.releaseIP(lastIP) - ip, err = alloc.acquireIP() + alloc.Release(lastIP) + ip, err = alloc.Acquire() if err != nil { t.Fatal(err) } From 09eacdfadec36f79cec74f037484b10ddfae6791 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 28 Feb 2013 11:51:14 -0800 Subject: [PATCH 22/35] Container can now take a list of ports to expose in its config --- container.go | 83 ++++++++++++++++++++++++++++--------------------- lxc_template.go | 2 +- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/container.go b/container.go index 7f5d592e81..aa28c7922b 100644 --- a/container.go +++ b/container.go @@ -11,6 +11,7 @@ import ( "os" "os/exec" "path" + "strconv" "strings" "syscall" "time" @@ -35,9 +36,9 @@ type Container struct { Filesystem *Filesystem State *State - network *NetworkInterface - networkAllocator *NetworkAllocator - NetworkConfig *NetworkConfig + network *NetworkInterface + networkManager *NetworkManager + NetworkSettings *NetworkSettings SysInitPath string lxcConfigPath string @@ -55,34 +56,36 @@ type Config struct { Hostname string User string Ram int64 + Ports []int Tty bool // Attach standard streams to a tty, including stdin if it is not closed. OpenStdin bool // Open stdin } -type NetworkConfig struct { +type NetworkSettings struct { IpAddress string IpPrefixLen int + Gateway string + PortMapping map[string]string } -func createContainer(id string, root string, command string, args []string, layers []string, config *Config, netAllocator *NetworkAllocator) (*Container, error) { +func createContainer(id string, root string, command string, args []string, layers []string, config *Config, netManager *NetworkManager) (*Container, error) { container := &Container{ - Id: id, - Root: root, - Created: time.Now(), - Path: command, - Args: args, - Config: config, - Filesystem: newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers), - State: newState(), - networkAllocator: netAllocator, - NetworkConfig: &NetworkConfig{}, - - SysInitPath: sysInitPath, - lxcConfigPath: path.Join(root, "config.lxc"), - stdout: newWriteBroadcaster(), - stderr: newWriteBroadcaster(), - stdoutLog: new(bytes.Buffer), - stderrLog: new(bytes.Buffer), + Id: id, + Root: root, + Created: time.Now(), + Path: command, + Args: args, + Config: config, + Filesystem: newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers), + State: newState(), + networkManager: netManager, + NetworkSettings: &NetworkSettings{}, + SysInitPath: sysInitPath, + lxcConfigPath: path.Join(root, "config.lxc"), + stdout: newWriteBroadcaster(), + stderr: newWriteBroadcaster(), + stdoutLog: new(bytes.Buffer), + stderrLog: new(bytes.Buffer), } if container.Config.OpenStdin { container.stdin, container.stdinPipe = io.Pipe() @@ -104,19 +107,19 @@ func createContainer(id string, root string, command string, args []string, laye return container, nil } -func loadContainer(containerPath string, netAllocator *NetworkAllocator) (*Container, error) { +func loadContainer(containerPath string, netManager *NetworkManager) (*Container, error) { data, err := ioutil.ReadFile(path.Join(containerPath, "config.json")) if err != nil { return nil, err } container := &Container{ - stdout: newWriteBroadcaster(), - stderr: newWriteBroadcaster(), - stdoutLog: new(bytes.Buffer), - stderrLog: new(bytes.Buffer), - lxcConfigPath: path.Join(containerPath, "config.lxc"), - networkAllocator: netAllocator, - NetworkConfig: &NetworkConfig{}, + stdout: newWriteBroadcaster(), + stderr: newWriteBroadcaster(), + stdoutLog: new(bytes.Buffer), + stderrLog: new(bytes.Buffer), + lxcConfigPath: path.Join(containerPath, "config.lxc"), + networkManager: netManager, + NetworkSettings: &NetworkSettings{}, } if err := json.Unmarshal(data, container); err != nil { return nil, err @@ -368,20 +371,30 @@ func (container *Container) StderrLog() io.Reader { } func (container *Container) allocateNetwork() error { - iface, err := container.networkAllocator.Allocate() + iface, err := container.networkManager.Allocate() if err != nil { return err } + container.NetworkSettings.PortMapping = make(map[string]string) + for _, port := range container.Config.Ports { + if extPort, err := iface.AllocatePort(port); err != nil { + iface.Release() + return err + } else { + container.NetworkSettings.PortMapping[strconv.Itoa(port)] = strconv.Itoa(extPort) + } + } container.network = iface - container.NetworkConfig.IpAddress = iface.IPNet.IP.String() - container.NetworkConfig.IpPrefixLen, _ = iface.IPNet.Mask.Size() + container.NetworkSettings.IpAddress = iface.IPNet.IP.String() + container.NetworkSettings.IpPrefixLen, _ = iface.IPNet.Mask.Size() + container.NetworkSettings.Gateway = iface.Gateway.String() return nil } func (container *Container) releaseNetwork() error { - err := container.networkAllocator.Release(container.network) + err := container.network.Release() container.network = nil - container.NetworkConfig = &NetworkConfig{} + container.NetworkSettings = &NetworkSettings{} return err } diff --git a/lxc_template.go b/lxc_template.go index 3e66885460..283391bfb3 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -19,7 +19,7 @@ lxc.network.flags = up lxc.network.link = lxcbr0 lxc.network.name = eth0 lxc.network.mtu = 1500 -lxc.network.ipv4 = {{.NetworkConfig.IpAddress}}/{{.NetworkConfig.IpPrefixLen}} +lxc.network.ipv4 = {{.NetworkSettings.IpAddress}}/{{.NetworkSettings.IpPrefixLen}} # root filesystem {{$ROOTFS := .Filesystem.RootFS}} From bd2f51290f4c34daec08474c0bd78952030be6ff Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 28 Feb 2013 11:52:07 -0800 Subject: [PATCH 23/35] Docker: Network manager integration --- docker.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docker.go b/docker.go index f44eb04059..67b67f7cdf 100644 --- a/docker.go +++ b/docker.go @@ -11,10 +11,10 @@ import ( ) type Docker struct { - root string - repository string - containers *list.List - networkAllocator *NetworkAllocator + root string + repository string + containers *list.List + networkManager *NetworkManager } func (docker *Docker) List() []*Container { @@ -52,7 +52,7 @@ func (docker *Docker) Create(id string, command string, args []string, layers [] return nil, fmt.Errorf("Container %v already exists", id) } root := path.Join(docker.repository, id) - container, err := createContainer(id, root, command, args, layers, config, docker.networkAllocator) + container, err := createContainer(id, root, command, args, layers, config, docker.networkManager) if err != nil { return nil, err } @@ -87,7 +87,7 @@ func (docker *Docker) restore() error { return err } for _, v := range dir { - container, err := loadContainer(path.Join(docker.repository, v.Name()), docker.networkAllocator) + container, err := loadContainer(path.Join(docker.repository, v.Name()), docker.networkManager) if err != nil { log.Printf("Failed to load container %v: %v", v.Name(), err) continue @@ -102,15 +102,15 @@ func New() (*Docker, error) { } func NewFromDirectory(root string) (*Docker, error) { - alloc, err := newNetworkAllocator(networkBridgeIface) + netManager, err := newNetworkManager(networkBridgeIface) if err != nil { return nil, err } docker := &Docker{ - root: root, - repository: path.Join(root, "containers"), - containers: list.New(), - networkAllocator: alloc, + root: root, + repository: path.Join(root, "containers"), + containers: list.New(), + networkManager: netManager, } if err := os.MkdirAll(docker.repository, 0700); err != nil && !os.IsExist(err) { From f857fa0dddbf48b959b0876582027ebc46dc5e9e Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 28 Feb 2013 11:52:22 -0800 Subject: [PATCH 24/35] Server: -p option to export TCP ports --- server/server.go | 55 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/server/server.go b/server/server.go index 2ef42f8b16..75acd8ddd3 100644 --- a/server/server.go +++ b/server/server.go @@ -11,10 +11,11 @@ import ( "github.com/dotcloud/docker/image" "github.com/dotcloud/docker/rcli" "io" - "net/http" + // "net/http" "net/url" "os" "path" + "strconv" "strings" "sync" "text/tabwriter" @@ -392,20 +393,20 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() - archive, err := future.Curl(u.String(), stdout) - if err != nil { - if resp, err := http.Get(u.String()); err != nil { - return err - } else { - archive = resp.Body - } - } - fmt.Fprintf(stdout, "Unpacking to %s\n", name) - img, err := srv.images.Import(name, archive, nil) - if err != nil { - return err - } - fmt.Fprintln(stdout, img.Id) + // archive, err := future.Curl(u.String(), stdout) + // if err != nil { + // if resp, err := http.Get(u.String()); err != nil { + // return err + // } else { + // archive = resp.Body + // } + // } + // fmt.Fprintf(stdout, "Unpacking to %s\n", name) + // img, err := srv.images.Import(name, archive, nil) + // if err != nil { + // return err + // } + // fmt.Fprintln(stdout, img.Id) return nil } @@ -679,10 +680,10 @@ func (srv *Server) CmdLogs(stdin io.ReadCloser, stdout io.Writer, args ...string return errors.New("No such container: " + cmd.Arg(0)) } -func (srv *Server) CreateContainer(img *image.Image, user string, tty bool, openStdin bool, comment string, cmd string, args ...string) (*docker.Container, error) { +func (srv *Server) CreateContainer(img *image.Image, ports []int, user string, tty bool, openStdin bool, comment string, cmd string, args ...string) (*docker.Container, error) { id := future.RandomId()[:8] container, err := srv.containers.Create(id, cmd, args, img.Layers, - &docker.Config{Hostname: id, User: user, Tty: tty, OpenStdin: openStdin}) + &docker.Config{Hostname: id, Ports: ports, User: user, Tty: tty, OpenStdin: openStdin}) if err != nil { return nil, err } @@ -743,6 +744,22 @@ func (srv *Server) CmdAttach(stdin io.ReadCloser, stdout io.Writer, args ...stri return nil } +// Ports type - Used to parse multiple -p flags +type ports []int + +func (p *ports) String() string { + return fmt.Sprint(*p) +} + +func (p *ports) Set(value string) error { + port, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("Invalid port: %v", value) + } + *p = append(*p, port) + return nil +} + func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "run", "[OPTIONS] IMAGE COMMAND [ARG...]", "Run a command in a new container") fl_user := cmd.String("u", "", "Username or UID") @@ -750,6 +767,8 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) fl_stdin := cmd.Bool("i", false, "Keep stdin open even if not attached") fl_tty := cmd.Bool("t", false, "Allocate a pseudo-tty") fl_comment := cmd.String("c", "", "Comment") + var fl_ports ports + cmd.Var(&fl_ports, "p", "Map a network port to the container") if err := cmd.Parse(args); err != nil { return nil } @@ -775,7 +794,7 @@ func (srv *Server) CmdRun(stdin io.ReadCloser, stdout io.Writer, args ...string) return errors.New("No such image: " + name) } // Create new container - container, err := srv.CreateContainer(img, *fl_user, *fl_tty, *fl_stdin, *fl_comment, cmdline[0], cmdline[1:]...) + container, err := srv.CreateContainer(img, fl_ports, *fl_user, *fl_tty, *fl_stdin, *fl_comment, cmdline[0], cmdline[1:]...) if err != nil { return errors.New("Error creating container: " + err.Error()) } From 2192d3371ceb7431a6dacd170ddadd4ef26d5783 Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 28 Feb 2013 11:57:57 -0800 Subject: [PATCH 25/35] Re-enabled lxc capabilities drop --- lxc_template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lxc_template.go b/lxc_template.go index 283391bfb3..931095c99d 100755 --- a/lxc_template.go +++ b/lxc_template.go @@ -82,7 +82,7 @@ lxc.mount.entry = /etc/resolv.conf {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 # drop linux capabilities (apply mainly to the user root in the container) -#lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw setfcap setpcap sys_admin sys_boot sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config +lxc.cap.drop = audit_control audit_write mac_admin mac_override mknod net_raw setfcap setpcap sys_admin sys_boot sys_module sys_nice sys_pacct sys_rawio sys_resource sys_time sys_tty_config # limits {{if .Config.Ram}} From 5675439b9100b40529b8eb87447231a2f5a01ad2 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 28 Feb 2013 16:30:31 -0800 Subject: [PATCH 26/35] Re-enabled CmdPull progress bar code which had been temporarily disabled --- server/server.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/server/server.go b/server/server.go index 75acd8ddd3..86fff3dcfe 100644 --- a/server/server.go +++ b/server/server.go @@ -11,7 +11,7 @@ import ( "github.com/dotcloud/docker/image" "github.com/dotcloud/docker/rcli" "io" - // "net/http" + "net/http" "net/url" "os" "path" @@ -393,20 +393,20 @@ func (srv *Server) CmdPull(stdin io.ReadCloser, stdout io.Writer, args ...string fmt.Fprintf(stdout, "Downloading from %s\n", u.String()) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() - // archive, err := future.Curl(u.String(), stdout) - // if err != nil { - // if resp, err := http.Get(u.String()); err != nil { - // return err - // } else { - // archive = resp.Body - // } - // } - // fmt.Fprintf(stdout, "Unpacking to %s\n", name) - // img, err := srv.images.Import(name, archive, nil) - // if err != nil { - // return err - // } - // fmt.Fprintln(stdout, img.Id) + archive, err := future.Curl(u.String(), stdout) + if err != nil { + if resp, err := http.Get(u.String()); err != nil { + return err + } else { + archive = resp.Body + } + } + fmt.Fprintf(stdout, "Unpacking to %s\n", name) + img, err := srv.images.Import(name, archive, nil) + if err != nil { + return err + } + fmt.Fprintln(stdout, img.Id) return nil } From d372dacbc9f4caf9743fe6c67c84c1e273faa663 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Fri, 1 Mar 2013 17:21:26 -0800 Subject: [PATCH 27/35] Updated install instructions and download links - New binary download links - Fixed incorrect install instructions - Merged client and host install instructions --- README.md | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index aea3d10e9d..95ade41d0b 100644 --- a/README.md +++ b/README.md @@ -149,8 +149,8 @@ Right now, the officially supported distributions are: Docker probably works on other distributions featuring a recent kernel, the AUFS patch, and up-to-date lxc. However this has not been tested. -Step by step host setup ------------------------ +Installation +--------------- 1. Set up your host of choice on a physical / virtual machine 2. Assume root identity on your newly installed environment (`sudo -s`) @@ -159,18 +159,14 @@ Step by step host setup apt-get update apt-get install lxc wget bsdtar curl -4. Download the latest version of the [docker binaries](https://dl.dropbox.com/u/20637798/docker.tar.gz) (`wget https://dl.dropbox.com/u/20637798/docker.tar.gz`) (warning: this may not be the most up-to-date build) -5. Extract the contents of the tar file `tar -xf docker.tar.gz` -6. Launch the docker daemon `./dockerd` -7. Download a base image by running 'docker pull base' +4. Download the latest docker binaries: `wget http://docker.io.s3.amazonaws.com/builds/$(uname -s)/$(uname -m)/docker-master.tgz` ([Or get the Linux/x86_64 binaries here](http://docker.io.s3.amazonaws.com/builds/Linux/x86_64/docker-master.tgz) ) +5. Extract the contents of the tar file `tar -xf docker-master.tar.gz` +6. Launch the docker daemon in the background `./dockerd &` +7. Download a base image `./docker pull base` +8. Run your first container! `./docker run -i -a -t base /bin/bash` +9. Start exploring `./docker --help` - -Client installation -------------------- - -4. Download the latest version of the [docker binaries](https://dl.dropbox.com/u/20637798/docker.tar.gz) (`wget https://dl.dropbox.com/u/20637798/docker.tar.gz`) -5. Extract the contents of the tar file `tar -xf docker.tar.gz` -6. You can now use the docker client binary `./docker`. Consider adding it to your `PATH` for simplicity. +Consider adding docker and dockerd to your `PATH` for simplicity. Vagrant Usage ------------- @@ -230,7 +226,7 @@ Welcome to Ubuntu 12.10 (GNU/Linux 3.5.0-17-generic x86_64) * Documentation: https://help.ubuntu.com/ Last login: Sun Feb 3 19:37:37 2013 -vagrant@vagrant-ubuntu-12:~$ DOCKER=localhost:4242 docker help +vagrant@vagrant-ubuntu-12:~$ docker help Usage: docker COMMAND [arg...] A self-sufficient runtime for linux containers. From 9f3b1a8ee05852fe41ddfba2b4765656d3e54805 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 5 Mar 2013 15:58:27 -0800 Subject: [PATCH 28/35] Updated README: networking no longer "coming soon" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 95ade41d0b..21d3ada2df 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Notable features * Resource isolation: system resources like cpu and memory can be allocated differently to each process container, using cgroups. -* Network isolation: each process container runs in its own network namespace, with a virtual interface and IP address of its own (COMING SOON) +* Network isolation: each process container runs in its own network namespace, with a virtual interface and IP address of its own. * Copy-on-write: root filesystems are created using copy-on-write, which makes deployment extremeley fast, memory-cheap and disk-cheap. From 711e29fb9b56e9583f9a2730d4d5db21e9cd2105 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 5 Mar 2013 16:00:17 -0800 Subject: [PATCH 29/35] Reorganized README Moved Install instructions closer to the top. --- README.md | 173 +++++++++++++++++++++++++++--------------------------- 1 file changed, 88 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 21d3ada2df..4f0d2440d5 100644 --- a/README.md +++ b/README.md @@ -34,47 +34,6 @@ Notable features * Interactive shell: docker can allocate a pseudo-tty and attach to the standard input of any container, for example to run a throaway interactive shell. -What is a Standard Container? ------------------------------ - -Docker defines a unit of software delivery called a Standard Container. The goal of a Standard Container is to encapsulate a software component and all its dependencies in -a format that is self-describing and portable, so that any compliant runtime can run it without extra dependency, regardless of the underlying machine and the contents of the container. - -The spec for Standard Containers is currently work in progress, but it is very straightforward. It mostly defines 1) an image format, 2) a set of standard operations, and 3) an execution environment. - -A great analogy for this is the shipping container. Just like Standard Containers are a fundamental unit of software delivery, shipping containers (http://bricks.argz.com/ins/7823-1/12) are a fundamental unit of physical delivery. - -### 1. STANDARD OPERATIONS - -Just like shipping containers, Standard Containers define a set of STANDARD OPERATIONS. Shipping containers can be lifted, stacked, locked, loaded, unloaded and labelled. Similarly, standard containers can be started, stopped, copied, snapshotted, downloaded, uploaded and tagged. - - -### 2. CONTENT-AGNOSTIC - -Just like shipping containers, Standard Containers are CONTENT-AGNOSTIC: all standard operations have the same effect regardless of the contents. A shipping container will be stacked in exactly the same way whether it contains Vietnamese powder coffe or spare Maserati parts. Similarly, Standard Containers are started or uploaded in the same way whether they contain a postgres database, a php application with its dependencies and application server, or Java build artifacts. - - -### 3. INFRASTRUCTURE-AGNOSTIC - -Both types of containers are INFRASTRUCTURE-AGNOSTIC: they can be transported to thousands of facilities around the world, and manipulated by a wide variety of equipment. A shipping container can be packed in a factory in Ukraine, transported by truck to the nearest routing center, stacked onto a train, loaded into a German boat by an Australian-built crane, stored in a warehouse at a US facility, etc. Similarly, a standard container can be bundled on my laptop, uploaded to S3, downloaded, run and snapshotted by a build server at Equinix in Virginia, uploaded to 10 staging servers in a home-made Openstack cluster, then sent to 30 production instances across 3 EC2 regions. - - -### 4. DESIGNED FOR AUTOMATION - -Because they offer the same standard operations regardless of content and infrastructure, Standard Containers, just like their physical counterpart, are extremely well-suited for automation. In fact, you could say automation is their secret weapon. - -Many things that once required time-consuming and error-prone human effort can now be programmed. Before shipping containers, a bag of powder coffee was hauled, dragged, dropped, rolled and stacked by 10 different people in 10 different locations by the time it reached its destination. 1 out of 50 disappeared. 1 out of 20 was damaged. The process was slow, inefficient and cost a fortune - and was entirely different depending on the facility and the type of goods. - -Similarly, before Standard Containers, by the time a software component ran in production, it had been individually built, configured, bundled, documented, patched, vendored, templated, tweaked and instrumented by 10 different people on 10 different computers. Builds failed, libraries conflicted, mirrors crashed, post-it notes were lost, logs were misplaced, cluster updates were half-broken. The process was slow, inefficient and cost a fortune - and was entirely different depending on the language and infrastructure provider. - - -### 5. INDUSTRIAL-GRADE DELIVERY - -There are 17 million shipping containers in existence, packed with every physical good imaginable. Every single one of them can be loaded on the same boats, by the same cranes, in the same facilities, and sent anywhere in the World with incredible efficiency. It is embarrassing to think that a 30 ton shipment of coffee can safely travel half-way across the World in *less time* than it takes a software team to deliver its code from one datacenter to another sitting 10 miles away. - -With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. - - Under the hood -------------- @@ -91,50 +50,6 @@ Under the hood, Docker is built on the following components: * [lxc](http://lxc.sourceforge.net/), a set of convenience scripts to simplify the creation of linux containers. -Standard Container Specification --------------------------------- - -(TODO) - -### Image format - - -### Standard operations - -* Copy -* Run -* Stop -* Wait -* Commit -* Attach standard streams -* List filesystem changes -* ... - -### Execution environment - -#### Root filesystem - -#### Environment variables - -#### Process arguments - -#### Networking - -#### Process namespacing - -#### Resource limits - -#### Process monitoring - -#### Logging - -#### Signals - -#### Pseudo-terminal allocation - -#### Security - - Setup instructions ================== @@ -249,3 +164,91 @@ Commands: attach Attach to a running container ``` + +What is a Standard Container? +----------------------------- + +Docker defines a unit of software delivery called a Standard Container. The goal of a Standard Container is to encapsulate a software component and all its dependencies in +a format that is self-describing and portable, so that any compliant runtime can run it without extra dependency, regardless of the underlying machine and the contents of the container. + +The spec for Standard Containers is currently work in progress, but it is very straightforward. It mostly defines 1) an image format, 2) a set of standard operations, and 3) an execution environment. + +A great analogy for this is the shipping container. Just like Standard Containers are a fundamental unit of software delivery, shipping containers (http://bricks.argz.com/ins/7823-1/12) are a fundamental unit of physical delivery. + +### 1. STANDARD OPERATIONS + +Just like shipping containers, Standard Containers define a set of STANDARD OPERATIONS. Shipping containers can be lifted, stacked, locked, loaded, unloaded and labelled. Similarly, standard containers can be started, stopped, copied, snapshotted, downloaded, uploaded and tagged. + + +### 2. CONTENT-AGNOSTIC + +Just like shipping containers, Standard Containers are CONTENT-AGNOSTIC: all standard operations have the same effect regardless of the contents. A shipping container will be stacked in exactly the same way whether it contains Vietnamese powder coffe or spare Maserati parts. Similarly, Standard Containers are started or uploaded in the same way whether they contain a postgres database, a php application with its dependencies and application server, or Java build artifacts. + + +### 3. INFRASTRUCTURE-AGNOSTIC + +Both types of containers are INFRASTRUCTURE-AGNOSTIC: they can be transported to thousands of facilities around the world, and manipulated by a wide variety of equipment. A shipping container can be packed in a factory in Ukraine, transported by truck to the nearest routing center, stacked onto a train, loaded into a German boat by an Australian-built crane, stored in a warehouse at a US facility, etc. Similarly, a standard container can be bundled on my laptop, uploaded to S3, downloaded, run and snapshotted by a build server at Equinix in Virginia, uploaded to 10 staging servers in a home-made Openstack cluster, then sent to 30 production instances across 3 EC2 regions. + + +### 4. DESIGNED FOR AUTOMATION + +Because they offer the same standard operations regardless of content and infrastructure, Standard Containers, just like their physical counterpart, are extremely well-suited for automation. In fact, you could say automation is their secret weapon. + +Many things that once required time-consuming and error-prone human effort can now be programmed. Before shipping containers, a bag of powder coffee was hauled, dragged, dropped, rolled and stacked by 10 different people in 10 different locations by the time it reached its destination. 1 out of 50 disappeared. 1 out of 20 was damaged. The process was slow, inefficient and cost a fortune - and was entirely different depending on the facility and the type of goods. + +Similarly, before Standard Containers, by the time a software component ran in production, it had been individually built, configured, bundled, documented, patched, vendored, templated, tweaked and instrumented by 10 different people on 10 different computers. Builds failed, libraries conflicted, mirrors crashed, post-it notes were lost, logs were misplaced, cluster updates were half-broken. The process was slow, inefficient and cost a fortune - and was entirely different depending on the language and infrastructure provider. + + +### 5. INDUSTRIAL-GRADE DELIVERY + +There are 17 million shipping containers in existence, packed with every physical good imaginable. Every single one of them can be loaded on the same boats, by the same cranes, in the same facilities, and sent anywhere in the World with incredible efficiency. It is embarrassing to think that a 30 ton shipment of coffee can safely travel half-way across the World in *less time* than it takes a software team to deliver its code from one datacenter to another sitting 10 miles away. + +With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. + + + + +Standard Container Specification +-------------------------------- + +(TODO) + +### Image format + + +### Standard operations + +* Copy +* Run +* Stop +* Wait +* Commit +* Attach standard streams +* List filesystem changes +* ... + +### Execution environment + +#### Root filesystem + +#### Environment variables + +#### Process arguments + +#### Networking + +#### Process namespacing + +#### Resource limits + +#### Process monitoring + +#### Logging + +#### Signals + +#### Pseudo-terminal allocation + +#### Security + + From 74c88fdbc0180c276f75322066e7da2e3cd38f6d Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 5 Mar 2013 22:39:43 -0800 Subject: [PATCH 30/35] docker rmi -r: remove all images matching a regexp --- image/image.go | 27 +++++++++++++++++++++++++++ server/server.go | 15 +++++++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/image/image.go b/image/image.go index ebc425db09..1060cf205f 100644 --- a/image/image.go +++ b/image/image.go @@ -9,6 +9,7 @@ import ( "os" "path" "path/filepath" + "regexp" "sort" "strings" "time" @@ -209,6 +210,32 @@ func (index *Index) Delete(name string) error { return nil } +// DeleteMatch deletes all images whose name matches `pattern` +func (index *Index) DeleteMatch(pattern string) error { + // Load + if err := index.load(); err != nil { + return err + } + for name, history := range index.ByName { + if match, err := regexp.MatchString(pattern, name); err != nil { + return err + } else if match { + fmt.Printf("Match: %s %s\n", name, pattern) + // Remove from index lookup + for _, image := range *history { + delete(index.ById, image.Id) + } + // Remove from name lookup + delete(index.ByName, name) + } + } + // Save + if err := index.save(); err != nil { + return err + } + return nil +} + func (index *Index) Names() []string { if err := index.load(); err != nil { return []string{} diff --git a/server/server.go b/server/server.go index 86fff3dcfe..93d53cd2b7 100644 --- a/server/server.go +++ b/server/server.go @@ -314,6 +314,7 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str // 'docker rmi NAME' removes all images with the name NAME func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") + fl_regexp := cmd.Bool("r", false, "Use IMAGE as a regular expression instead of an exact name") if err := cmd.Parse(args); err != nil { cmd.Usage() return nil @@ -323,11 +324,17 @@ func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) return nil } for _, name := range cmd.Args() { - image := srv.images.Find(name) - if image == nil { - return errors.New("No such image: " + name) + var err error + if *fl_regexp { + err = srv.images.DeleteMatch(name) + } else { + image := srv.images.Find(name) + if image == nil { + return errors.New("No such image: " + name) + } + err = srv.images.Delete(name) } - if err := srv.images.Delete(name); err != nil { + if err != nil { return err } } From 836e7b2881b9257ecb31fc3bbd696af84e17730d Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 5 Mar 2013 22:44:09 -0800 Subject: [PATCH 31/35] Moved Vagrant guide to the wiki Trying to keep the authoritative docs very small, so we can keep it correct and up-to-date. --- README.md | 81 ------------------------------------------------------- 1 file changed, 81 deletions(-) diff --git a/README.md b/README.md index 4f0d2440d5..8a4a78b1e3 100644 --- a/README.md +++ b/README.md @@ -83,87 +83,6 @@ Installation Consider adding docker and dockerd to your `PATH` for simplicity. -Vagrant Usage -------------- - -1. Install Vagrant from http://vagrantup.com -2. Run `vagrant up`. This will take a few minutes as it does the following: - - Download Quantal64 base box - - Kick off Puppet to do: - - Download & untar most recent docker binary tarball to vagrant homedir. - - Debootstrap to /var/lib/docker/images/ubuntu. - - Install & run dockerd as service. - - Put docker in /usr/local/bin. - - Put latest Go toolchain in /usr/local/go. - -Sample run output: - -```bash -$ vagrant up -[default] Importing base box 'quantal64'... -[default] Matching MAC address for NAT networking... -[default] Clearing any previously set forwarded ports... -[default] Forwarding ports... -[default] -- 22 => 2222 (adapter 1) -[default] Creating shared folders metadata... -[default] Clearing any previously set network interfaces... -[default] Booting VM... -[default] Waiting for VM to boot. This can take a few minutes. -[default] VM booted and ready for use! -[default] Mounting shared folders... -[default] -- v-root: /vagrant -[default] -- manifests: /tmp/vagrant-puppet/manifests -[default] -- v-pp-m0: /tmp/vagrant-puppet/modules-0 -[default] Running provisioner: Vagrant::Provisioners::Puppet... -[default] Running Puppet with /tmp/vagrant-puppet/manifests/quantal64.pp... -stdin: is not a tty -notice: /Stage[main]//Node[default]/Exec[apt_update]/returns: executed successfully - -notice: /Stage[main]/Docker/Exec[fetch-docker]/returns: executed successfully -notice: /Stage[main]/Docker/Package[lxc]/ensure: ensure changed 'purged' to 'present' -notice: /Stage[main]/Docker/Exec[fetch-go]/returns: executed successfully - -notice: /Stage[main]/Docker/Exec[copy-docker-bin]/returns: executed successfully -notice: /Stage[main]/Docker/Exec[debootstrap]/returns: executed successfully -notice: /Stage[main]/Docker/File[/etc/init/dockerd.conf]/ensure: defined content as '{md5}78a593d38dd9919af14d8f0545ac95e9' - -notice: /Stage[main]/Docker/Service[dockerd]/ensure: ensure changed 'stopped' to 'running' - -notice: Finished catalog run in 329.74 seconds -``` - -When this has successfully completed, you should be able to get into your new system with `vagrant ssh` and use `docker`: - -```bash -$ vagrant ssh -Welcome to Ubuntu 12.10 (GNU/Linux 3.5.0-17-generic x86_64) - - * Documentation: https://help.ubuntu.com/ - -Last login: Sun Feb 3 19:37:37 2013 -vagrant@vagrant-ubuntu-12:~$ docker help -Usage: docker COMMAND [arg...] - -A self-sufficient runtime for linux containers. - -Commands: - run Run a command in a container - ps Display a list of containers - pull Download a tarball and create a container from it - put Upload a tarball and create a container from it - rm Remove containers - wait Wait for the state of a container to change - stop Stop a running container - logs Fetch the logs of a container - diff Inspect changes on a container's filesystem - commit Save the state of a container - attach Attach to the standard inputs and outputs of a running container - info Display system-wide information - tar Stream the contents of a container as a tar archive - web Generate a web UI - attach Attach to a running container -``` - What is a Standard Container? ----------------------------- From 791592731eccfae0072b20a80d15ec31e34dc88f Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 6 Mar 2013 00:37:00 -0800 Subject: [PATCH 32/35] Remove leftover debug message --- image/image.go | 1 - 1 file changed, 1 deletion(-) diff --git a/image/image.go b/image/image.go index 1060cf205f..4702919ab4 100644 --- a/image/image.go +++ b/image/image.go @@ -220,7 +220,6 @@ func (index *Index) DeleteMatch(pattern string) error { if match, err := regexp.MatchString(pattern, name); err != nil { return err } else if match { - fmt.Printf("Match: %s %s\n", name, pattern) // Remove from index lookup for _, image := range *history { delete(index.ById, image.Id) From 410ebe379dbf60d8fd75ba695e4e8036fdba6b65 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 6 Mar 2013 00:39:03 -0800 Subject: [PATCH 33/35] docker port: lookup public-facing tcp ports for a container --- server/server.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/server/server.go b/server/server.go index 93d53cd2b7..bc642cdbb0 100644 --- a/server/server.go +++ b/server/server.go @@ -44,6 +44,7 @@ func (srv *Server) Help() string { {"ps", "Display a list of containers"}, {"pull", "Download a tarball and create a container from it"}, {"put", "Upload a tarball and create a container from it"}, + {"port", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT"}, {"rm", "Remove containers"}, {"kill", "Kill a running container"}, {"wait", "Wait for the state of a container to change"}, @@ -311,6 +312,30 @@ func (srv *Server) CmdInspect(stdin io.ReadCloser, stdout io.Writer, args ...str return nil } +func (srv *Server) CmdPort(stdin io.ReadCloser, stdout io.Writer, args ...string) error { + cmd := rcli.Subcmd(stdout, "port", "[OPTIONS] CONTAINER PRIVATE_PORT", "Lookup the public-facing port which is NAT-ed to PRIVATE_PORT") + if err := cmd.Parse(args); err != nil { + cmd.Usage() + return nil + } + if cmd.NArg() != 2 { + cmd.Usage() + return nil + } + name := cmd.Arg(0) + privatePort := cmd.Arg(1) + if container := srv.containers.Get(name); container == nil { + return errors.New("No such container: " + name) + } else { + if frontend, exists := container.NetworkSettings.PortMapping[privatePort]; !exists { + return fmt.Errorf("No private port '%s' allocated on %s", privatePort, name) + } else { + fmt.Fprintln(stdout, frontend) + } + } + return nil +} + // 'docker rmi NAME' removes all images with the name NAME func (srv *Server) CmdRmi(stdin io.ReadCloser, stdout io.Writer, args ...string) error { cmd := rcli.Subcmd(stdout, "rmimage", "[OPTIONS] IMAGE", "Remove an image") From 2df0bc6bc0df53c7657404a92fb8f256b7109de0 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 6 Mar 2013 16:08:44 -0800 Subject: [PATCH 34/35] Container logs are persisted on disk --- container.go | 55 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/container.go b/container.go index aa28c7922b..530c08bc39 100644 --- a/container.go +++ b/container.go @@ -1,7 +1,6 @@ package docker import ( - "bytes" "encoding/json" "errors" "github.com/kr/pty" @@ -12,7 +11,6 @@ import ( "os/exec" "path" "strconv" - "strings" "syscall" "time" ) @@ -48,8 +46,8 @@ type Container struct { stdin io.ReadCloser stdinPipe io.WriteCloser - stdoutLog *bytes.Buffer - stderrLog *bytes.Buffer + stdoutLog *os.File + stderrLog *os.File } type Config struct { @@ -84,8 +82,20 @@ func createContainer(id string, root string, command string, args []string, laye lxcConfigPath: path.Join(root, "config.lxc"), stdout: newWriteBroadcaster(), stderr: newWriteBroadcaster(), - stdoutLog: new(bytes.Buffer), - stderrLog: new(bytes.Buffer), + } + if err := os.Mkdir(root, 0700); err != nil { + return nil, err + } + // Setup logging of stdout and stderr to disk + if stdoutLog, err := os.OpenFile(path.Join(container.Root, id+"-stdout.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { + return nil, err + } else { + container.stdoutLog = stdoutLog + } + if stderrLog, err := os.OpenFile(path.Join(container.Root, id+"-stderr.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { + return nil, err + } else { + container.stderrLog = stderrLog } if container.Config.OpenStdin { container.stdin, container.stdinPipe = io.Pipe() @@ -95,9 +105,6 @@ func createContainer(id string, root string, command string, args []string, laye container.stdout.AddWriter(NopWriteCloser(container.stdoutLog)) container.stderr.AddWriter(NopWriteCloser(container.stderrLog)) - if err := os.Mkdir(root, 0700); err != nil { - return nil, err - } if err := container.Filesystem.createMountPoints(); err != nil { return nil, err } @@ -115,15 +122,29 @@ func loadContainer(containerPath string, netManager *NetworkManager) (*Container container := &Container{ stdout: newWriteBroadcaster(), stderr: newWriteBroadcaster(), - stdoutLog: new(bytes.Buffer), - stderrLog: new(bytes.Buffer), lxcConfigPath: path.Join(containerPath, "config.lxc"), networkManager: netManager, NetworkSettings: &NetworkSettings{}, } + // Load container settings if err := json.Unmarshal(data, container); err != nil { return nil, err } + // Setup logging of stdout and stderr to disk + if stdoutLog, err := os.OpenFile(path.Join(container.Root, container.Id+"-stdout.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { + return nil, err + } else { + container.stdoutLog = stdoutLog + } + if stderrLog, err := os.OpenFile(path.Join(container.Root, container.Id+"-stderr.log"), os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600); err != nil { + return nil, err + } else { + container.stderrLog = stderrLog + } + container.stdout.AddWriter(NopWriteCloser(container.stdoutLog)) + container.stderr.AddWriter(NopWriteCloser(container.stderrLog)) + + // Create mountpoints if err := container.Filesystem.createMountPoints(); err != nil { return nil, err } @@ -357,7 +378,11 @@ func (container *Container) StdoutPipe() (io.ReadCloser, error) { } func (container *Container) StdoutLog() io.Reader { - return strings.NewReader(container.stdoutLog.String()) + r, err := os.Open(container.stdoutLog.Name()) + if err != nil { + return nil + } + return r } func (container *Container) StderrPipe() (io.ReadCloser, error) { @@ -367,7 +392,11 @@ func (container *Container) StderrPipe() (io.ReadCloser, error) { } func (container *Container) StderrLog() io.Reader { - return strings.NewReader(container.stderrLog.String()) + r, err := os.Open(container.stderrLog.Name()) + if err != nil { + return nil + } + return r } func (container *Container) allocateNetwork() error { From fb350e0c7705850cc78e1dc1dc63b56aca06c3cc Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Thu, 7 Mar 2013 09:25:41 -0800 Subject: [PATCH 35/35] Setup a predictable, repeatable environment for containers --- container_test.go | 50 +++++++++++++++++++++++++++++++++++++++++++++++ sysinit.go | 8 ++++++++ 2 files changed, 58 insertions(+) diff --git a/container_test.go b/container_test.go index 17ec85c6b1..8c187ecb97 100644 --- a/container_test.go +++ b/container_test.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "io/ioutil" + "sort" "strings" "testing" "time" @@ -511,6 +512,55 @@ func TestTty(t *testing.T) { } } +func TestEnv(t *testing.T) { + docker, err := newTestDocker() + if err != nil { + t.Fatal(err) + } + container, err := docker.Create( + "env_test", + "/usr/bin/env", + []string{}, + []string{testLayerPath}, + &Config{}, + ) + if err != nil { + t.Fatal(err) + } + defer docker.Destroy(container) + stdout, err := container.StdoutPipe() + if err != nil { + t.Fatal(err) + } + defer stdout.Close() + if err := container.Start(); err != nil { + t.Fatal(err) + } + container.Wait() + output, err := ioutil.ReadAll(stdout) + if err != nil { + t.Fatal(err) + } + actualEnv := strings.Split(string(output), "\n") + if actualEnv[len(actualEnv)-1] == "" { + actualEnv = actualEnv[:len(actualEnv)-1] + } + sort.Strings(actualEnv) + goodEnv := []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "HOME=/", + } + sort.Strings(goodEnv) + if len(goodEnv) != len(actualEnv) { + t.Fatalf("Wrong environment: should be %d variables, not: '%s'\n", len(goodEnv), strings.Join(actualEnv, ", ")) + } + for i := range goodEnv { + if actualEnv[i] != goodEnv[i] { + t.Fatalf("Wrong environment variable: should be %s, not %s", goodEnv[i], actualEnv[i]) + } + } +} + func BenchmarkRunSequencial(b *testing.B) { docker, err := newTestDocker() if err != nil { diff --git a/sysinit.go b/sysinit.go index f701417978..c475b3365d 100644 --- a/sysinit.go +++ b/sysinit.go @@ -52,6 +52,13 @@ func changeUser(u string) { } } +// Set the environment to a known, repeatable state +func setupEnv() { + os.Clearenv() + os.Setenv("HOME", "/") + os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") +} + func executeProgram(name string, args []string) { path, err := exec.LookPath(name) if err != nil { @@ -79,5 +86,6 @@ func SysInit() { setupNetworking(*gw) changeUser(*u) + setupEnv() executeProgram(flag.Arg(0), flag.Args()) }