Integrated the network allocator into Docker. A networking environment

is assigned to each container upon Start and released whenever the
container exits.

container.go

@@ -33,9 +33,12 @@ type Container struct {
 
 	Config     *Config
 	Filesystem *Filesystem
-	Network    *NetworkInterface
 	State      *State
 
+	network          *NetworkInterface
+	networkAllocator *NetworkAllocator
+	NetworkConfig    *NetworkConfig
+
 	SysInitPath   string
 	lxcConfigPath string
 	cmd           *exec.Cmd
@@ -56,16 +59,23 @@ type Config struct {
 	OpenStdin bool // Open stdin
 }
 
-func createContainer(id string, root string, command string, args []string, layers []string, config *Config) (*Container, error) {
+type NetworkConfig struct {
+	IpAddress   string
+	IpPrefixLen int
+}
+
+func createContainer(id string, root string, command string, args []string, layers []string, config *Config, netAllocator *NetworkAllocator) (*Container, error) {
 	container := &Container{
-		Id:         id,
+		Id:               id,
-		Root:       root,
+		Root:             root,
-		Created:    time.Now(),
+		Created:          time.Now(),
-		Path:       command,
+		Path:             command,
-		Args:       args,
+		Args:             args,
-		Config:     config,
+		Config:           config,
-		Filesystem: newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers),
+		Filesystem:       newFilesystem(path.Join(root, "rootfs"), path.Join(root, "rw"), layers),
-		State:      newState(),
+		State:            newState(),
+		networkAllocator: netAllocator,
+		NetworkConfig:    &NetworkConfig{},
 
 		SysInitPath:   sysInitPath,
 		lxcConfigPath: path.Join(root, "config.lxc"),
@@ -88,27 +98,25 @@ func createContainer(id string, root string, command string, args []string, laye
 	if err := container.Filesystem.createMountPoints(); err != nil {
 		return nil, err
 	}
-	var err error
-	if container.Network, err = allocateNetwork(); err != nil {
-		return nil, err
-	}
 	if err := container.save(); err != nil {
 		return nil, err
 	}
 	return container, nil
 }
 
-func loadContainer(containerPath string) (*Container, error) {
+func loadContainer(containerPath string, netAllocator *NetworkAllocator) (*Container, error) {
 	data, err := ioutil.ReadFile(path.Join(containerPath, "config.json"))
 	if err != nil {
 		return nil, err
 	}
 	container := &Container{
-		stdout:        newWriteBroadcaster(),
+		stdout:           newWriteBroadcaster(),
-		stderr:        newWriteBroadcaster(),
+		stderr:           newWriteBroadcaster(),
-		stdoutLog:     new(bytes.Buffer),
+		stdoutLog:        new(bytes.Buffer),
-		stderrLog:     new(bytes.Buffer),
+		stderrLog:        new(bytes.Buffer),
-		lxcConfigPath: path.Join(containerPath, "config.lxc"),
+		lxcConfigPath:    path.Join(containerPath, "config.lxc"),
+		networkAllocator: netAllocator,
+		NetworkConfig:    &NetworkConfig{},
 	}
 	if err := json.Unmarshal(data, container); err != nil {
 		return nil, err
@@ -268,6 +276,9 @@ func (container *Container) Start() error {
 	if err := container.Filesystem.EnsureMounted(); err != nil {
 		return err
 	}
+	if err := container.allocateNetwork(); err != nil {
+		return err
+	}
 	if err := container.generateLXCConfig(); err != nil {
 		return err
 	}
@@ -279,7 +290,7 @@ func (container *Container) Start() error {
 	}
 
 	// Networking
-	params = append(params, "-g", container.Network.Gateway.String())
+	params = append(params, "-g", container.network.Gateway.String())
 
 	// User
 	if container.Config.User != "" {
@@ -356,12 +367,33 @@ func (container *Container) StderrLog() io.Reader {
 	return strings.NewReader(container.stderrLog.String())
 }
 
+func (container *Container) allocateNetwork() error {
+	iface, err := container.networkAllocator.Allocate()
+	if err != nil {
+		return err
+	}
+	container.network = iface
+	container.NetworkConfig.IpAddress = iface.IPNet.IP.String()
+	container.NetworkConfig.IpPrefixLen, _ = iface.IPNet.Mask.Size()
+	return nil
+}
+
+func (container *Container) releaseNetwork() error {
+	err := container.networkAllocator.Release(container.network)
+	container.network = nil
+	container.NetworkConfig = &NetworkConfig{}
+	return err
+}
+
 func (container *Container) monitor() {
 	// Wait for the program to exit
 	container.cmd.Wait()
 	exitCode := container.cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
 
 	// Cleanup
+	if err := container.releaseNetwork(); err != nil {
+		log.Printf("%v: Failed to release network: %v", container.Id, err)
+	}
 	container.stdout.Close()
 	container.stderr.Close()
 	if err := container.Filesystem.Umount(); err != nil {
@@ -429,7 +461,6 @@ func (container *Container) Restart() error {
 }
 
 func (container *Container) Wait() {
-
 	for container.State.Running {
 		container.State.wait()
 	}

docker.go

@@ -11,9 +11,10 @@ import (
 )
 
 type Docker struct {
-	root       string
+	root             string
-	repository string
+	repository       string
-	containers *list.List
+	containers       *list.List
+	networkAllocator *NetworkAllocator
 }
 
 func (docker *Docker) List() []*Container {
@@ -51,7 +52,7 @@ func (docker *Docker) Create(id string, command string, args []string, layers []
 		return nil, fmt.Errorf("Container %v already exists", id)
 	}
 	root := path.Join(docker.repository, id)
-	container, err := createContainer(id, root, command, args, layers, config)
+	container, err := createContainer(id, root, command, args, layers, config, docker.networkAllocator)
 	if err != nil {
 		return nil, err
 	}
@@ -86,7 +87,7 @@ func (docker *Docker) restore() error {
 		return err
 	}
 	for _, v := range dir {
-		container, err := loadContainer(path.Join(docker.repository, v.Name()))
+		container, err := loadContainer(path.Join(docker.repository, v.Name()), docker.networkAllocator)
 		if err != nil {
 			log.Printf("Failed to load container %v: %v", v.Name(), err)
 			continue
@@ -101,10 +102,15 @@ func New() (*Docker, error) {
 }
 
 func NewFromDirectory(root string) (*Docker, error) {
+	alloc, err := newNetworkAllocator(networkBridgeIface)
+	if err != nil {
+		return nil, err
+	}
 	docker := &Docker{
-		root:       root,
+		root:             root,
-		repository: path.Join(root, "containers"),
+		repository:       path.Join(root, "containers"),
-		containers: list.New(),
+		containers:       list.New(),
+		networkAllocator: alloc,
 	}
 
 	if err := os.MkdirAll(docker.repository, 0700); err != nil && !os.IsExist(err) {

lxc_template.go

@@ -19,7 +19,7 @@ lxc.network.flags = up
 lxc.network.link = lxcbr0
 lxc.network.name = eth0
 lxc.network.mtu = 1500
-lxc.network.ipv4 = {{.Network.IpAddress}}/{{.Network.IpPrefixLen}}
+lxc.network.ipv4 = {{.NetworkConfig.IpAddress}}/{{.NetworkConfig.IpPrefixLen}}
 
 # root filesystem
 {{$ROOTFS := .Filesystem.RootFS}}

network.go

@@ -5,7 +5,6 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
-	"math/rand"
 	"net"
 )
 
@@ -14,11 +13,12 @@ const (
 )
 
 type NetworkInterface struct {
-	IpAddress   string
+	IPNet   net.IPNet
-	IpPrefixLen int
+	Gateway net.IP
-	Gateway     net.IP
 }
 
+// IP utils
+
 func networkRange(network *net.IPNet) (net.IP, net.IP) {
 	netIP := network.IP.To4()
 	firstIP := netIP.Mask(network.Mask)
@@ -51,10 +51,11 @@ func intToIp(n int32) (net.IP, error) {
 }
 
 func networkSize(mask net.IPMask) (int32, error) {
+	m := net.IPv4Mask(0, 0, 0, 0)
 	for i := 0; i < net.IPv4len; i++ {
-		mask[i] = ^mask[i]
+		m[i] = ^mask[i]
 	}
-	buf := bytes.NewBuffer(mask)
+	buf := bytes.NewBuffer(m)
 	var n int32
 	if err := binary.Read(buf, binary.BigEndian, &n); err != nil {
 		return 0, err
@@ -62,21 +63,7 @@ func networkSize(mask net.IPMask) (int32, error) {
 	return n + 1, nil
 }
 
-func allocateIPAddress(network *net.IPNet) (net.IP, error) {
+func getIfaceAddr(name string) (net.Addr, error) {
-	ip, _ := networkRange(network)
-	netSize, err := networkSize(network.Mask)
-	if err != nil {
-		return net.IP{}, err
-	}
-	numIp, err := ipToInt(ip)
-	if err != nil {
-		return net.IP{}, err
-	}
-	numIp += rand.Int31n(netSize)
-	return intToIp(numIp)
-}
-
-func getBridgeAddr(name string) (net.Addr, error) {
 	iface, err := net.InterfaceByName(name)
 	if err != nil {
 		return nil, err
@@ -101,31 +88,31 @@ func getBridgeAddr(name string) (net.Addr, error) {
 	return addrs4[0], nil
 }
 
-func allocateNetwork() (*NetworkInterface, error) {
+// Network allocator
-	bridgeAddr, err := getBridgeAddr(networkBridgeIface)
+func newNetworkAllocator(iface string) (*NetworkAllocator, error) {
+	addr, err := getIfaceAddr(iface)
 	if err != nil {
 		return nil, err
 	}
-	bridge := bridgeAddr.(*net.IPNet)
+	network := addr.(*net.IPNet)
-	ipPrefixLen, _ := bridge.Mask.Size()
+
-	ip, err := allocateIPAddress(bridge)
+	alloc := &NetworkAllocator{
-	if err != nil {
+		iface: iface,
+		net:   network,
+	}
+	if err := alloc.populateFromNetwork(network); err != nil {
 		return nil, err
 	}
-	iface := &NetworkInterface{
+	return alloc, nil
-		IpAddress:   ip.String(),
-		IpPrefixLen: ipPrefixLen,
-		Gateway:     bridge.IP,
-	}
-	return iface, nil
 }
 
 type NetworkAllocator struct {
 	iface string
+	net   *net.IPNet
 	queue chan (net.IP)
 }
 
-func (alloc *NetworkAllocator) Acquire() (net.IP, error) {
+func (alloc *NetworkAllocator) acquireIP() (net.IP, error) {
 	select {
 	case ip := <-alloc.queue:
 		return ip, nil
@@ -135,7 +122,7 @@ func (alloc *NetworkAllocator) Acquire() (net.IP, error) {
 	return net.IP{}, nil
 }
 
-func (alloc *NetworkAllocator) Release(ip net.IP) error {
+func (alloc *NetworkAllocator) releaseIP(ip net.IP) error {
 	select {
 	case alloc.queue <- ip:
 		return nil
@@ -145,7 +132,7 @@ func (alloc *NetworkAllocator) Release(ip net.IP) error {
 	return nil
 }
 
-func (alloc *NetworkAllocator) PopulateFromNetwork(network *net.IPNet) error {
+func (alloc *NetworkAllocator) populateFromNetwork(network *net.IPNet) error {
 	firstIP, _ := networkRange(network)
 	size, err := networkSize(network.Mask)
 	if err != nil {
@@ -168,16 +155,24 @@ func (alloc *NetworkAllocator) PopulateFromNetwork(network *net.IPNet) error {
 		if ip.Equal(network.IP) {
 			continue
 		}
-		alloc.Release(ip)
+		alloc.releaseIP(ip)
 	}
 	return nil
 }
 
-func (alloc *NetworkAllocator) PopulateFromInterface(iface string) error {
+func (alloc *NetworkAllocator) Allocate() (*NetworkInterface, error) {
-	addr, err := getBridgeAddr(iface)
+	// ipPrefixLen, _ := alloc.net.Mask.Size()
+	ip, err := alloc.acquireIP()
 	if err != nil {
-		return err
+		return nil, err
 	}
-	network := addr.(*net.IPNet)
+	iface := &NetworkInterface{
-	return alloc.PopulateFromNetwork(network)
+		IPNet:   net.IPNet{ip, alloc.net.Mask},
+		Gateway: alloc.net.IP,
+	}
+	return iface, nil
+}
+
+func (alloc *NetworkAllocator) Release(iface *NetworkInterface) error {
+	return alloc.releaseIP(iface.IPNet.IP)
 }

network_test.go

@@ -103,22 +103,22 @@ func TestConversion(t *testing.T) {
 func TestNetworkAllocator(t *testing.T) {
 	alloc := NetworkAllocator{}
 	_, n, _ := net.ParseCIDR("127.0.0.1/29")
-	alloc.PopulateFromNetwork(n)
+	alloc.populateFromNetwork(n)
 	var lastIP net.IP
 	for i := 0; i < 5; i++ {
-		ip, err := alloc.Acquire()
+		ip, err := alloc.acquireIP()
 		if err != nil {
 			t.Fatal(err)
 		}
 		lastIP = ip
 	}
-	ip, err := alloc.Acquire()
+	ip, err := alloc.acquireIP()
 	if err == nil {
 		t.Fatal("There shouldn't be any IP addresses at this point")
 	}
 	// Release 1 IP
-	alloc.Release(lastIP)
+	alloc.releaseIP(lastIP)
-	ip, err = alloc.Acquire()
+	ip, err = alloc.acquireIP()
 	if err != nil {
 		t.Fatal(err)
 	}