From 9e2e8a923c28833dd631a7206d3664a6a90cc24c Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 22 Jan 2014 15:52:39 -0800 Subject: [PATCH 01/12] WIP for ip allocator Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- network.go | 122 +----------- networkdriver/portallocator/allocator.go | 229 +++++++++++++++++++++++ 2 files changed, 230 insertions(+), 121 deletions(-) create mode 100644 networkdriver/portallocator/allocator.go diff --git a/network.go b/network.go index 1139f6ad37..473df0ffdb 100644 --- a/network.go +++ b/network.go @@ -1,7 +1,6 @@ package docker import ( - "encoding/binary" "errors" "fmt" "github.com/dotcloud/docker/pkg/iptables" @@ -25,61 +24,6 @@ const ( siocBRADDBR = 0x89a0 ) -// Calculates the first and last IP addresses in an IPNet -func networkRange(network *net.IPNet) (net.IP, net.IP) { - netIP := network.IP.To4() - firstIP := netIP.Mask(network.Mask) - lastIP := net.IPv4(0, 0, 0, 0).To4() - for i := 0; i < len(lastIP); i++ { - lastIP[i] = netIP[i] | ^network.Mask[i] - } - return firstIP, lastIP -} - -// Detects overlap between one IPNet and another -func networkOverlaps(netX *net.IPNet, netY *net.IPNet) bool { - firstIP, _ := networkRange(netX) - if netY.Contains(firstIP) { - return true - } - firstIP, _ = networkRange(netY) - if netX.Contains(firstIP) { - return true - } - return false -} - -// Converts a 4 bytes IP into a 32 bit integer -func ipToInt(ip net.IP) int32 { - return int32(binary.BigEndian.Uint32(ip.To4())) -} - -// Converts 32 bit integer into a 4 bytes IP address -func intToIP(n int32) net.IP { - b := make([]byte, 4) - binary.BigEndian.PutUint32(b, uint32(n)) - return net.IP(b) -} - -// Given a netmask, calculates the number of available hosts -func networkSize(mask net.IPMask) int32 { - m := net.IPv4Mask(0, 0, 0, 0) - for i := 0; i < net.IPv4len; i++ { - m[i] = ^mask[i] - } - - return int32(binary.BigEndian.Uint32(m)) + 1 -} - -func checkRouteOverlaps(networks []netlink.Route, dockerNetwork *net.IPNet) error { - for _, network := range networks { - if network.IPNet != nil && networkOverlaps(dockerNetwork, network.IPNet) { - return fmt.Errorf("Network %s is already routed: '%s'", dockerNetwork, network) - } - } - return nil -} - func checkNameserverOverlaps(nameservers []string, dockerNetwork *net.IPNet) error { if len(nameservers) > 0 { for _, ns := range nameservers { @@ -142,10 +86,7 @@ func CreateBridgeIface(config *DaemonConfig) error { if err != nil { return err } - routes, err := netlink.NetworkGetRoutes() - if err != nil { - return err - } + // TODO: @crosbymichael register route if err := checkRouteOverlaps(routes, dockerNetwork); err == nil { if err := checkNameserverOverlaps(nameservers, dockerNetwork); err == nil { ifaceAddr = addr @@ -441,67 +382,6 @@ type allocatedIP struct { } func (alloc *IPAllocator) run() { - firstIP, _ := networkRange(alloc.network) - ipNum := ipToInt(firstIP) - ownIP := ipToInt(alloc.network.IP) - size := networkSize(alloc.network.Mask) - - pos := int32(1) - max := size - 2 // -1 for the broadcast address, -1 for the gateway address - for { - var ( - newNum int32 - inUse bool - ) - - // Find first unused IP, give up after one whole round - for attempt := int32(0); attempt < max; attempt++ { - newNum = ipNum + pos - - pos = pos%max + 1 - - // The network's IP is never okay to use - if newNum == ownIP { - continue - } - - if _, inUse = alloc.inUse[newNum]; !inUse { - // We found an unused IP - break - } - } - - ip := allocatedIP{ip: intToIP(newNum)} - if inUse { - ip.err = errors.New("No unallocated IP available") - } - - select { - case quit := <-alloc.quit: - if quit { - return - } - case alloc.queueAlloc <- ip: - alloc.inUse[newNum] = struct{}{} - case released := <-alloc.queueReleased: - r := ipToInt(released) - delete(alloc.inUse, r) - - if inUse { - // If we couldn't allocate a new IP, the released one - // will be the only free one now, so instantly use it - // next time - pos = r - ipNum - } else { - // Use same IP as last time - if pos == 1 { - pos = max - } else { - pos-- - } - } - } - } } func (alloc *IPAllocator) Acquire() (net.IP, error) { diff --git a/networkdriver/portallocator/allocator.go b/networkdriver/portallocator/allocator.go new file mode 100644 index 0000000000..5d8e812309 --- /dev/null +++ b/networkdriver/portallocator/allocator.go @@ -0,0 +1,229 @@ +package ipallocator + +import ( + "encoding/binary" + "errors" + "github.com/dotcloud/docker/pkg/netlink" + "net" + "sync" +) + +type networkSet map[iPNet]iPSet +type iPSet map[string]struct{} + +type iPNet struct { + IP string + Mask string +} + +var ( + ErrNetworkAlreadyAllocated = errors.New("requested network overlaps with existing network") + ErrNetworkAlreadyRegisterd = errors.New("requested network is already registered") + lock = sync.Mutex{} + allocatedIPs = networkSet{} + availableIPS = networkSet{} +) + +func RegisterNetwork(network *net.IPNet) error { + lock.Lock() + defer lock.Unlock() + + routes, err := netlink.NetworkGetRoutes() + if err != nil { + return err + } + + if err := checkRouteOverlaps(routes, network); err != nil { + return err + } + + if err := checkExistingNetworkOverlaps(network); err != nil { + return err + } + + allocatedIPs[newIPNet(network)] = iPSet{} + + return nil +} + +func RequestIP(ip *net.IPAddr) (*net.IPAddr, error) { + lock.Lock() + defer lock.Unlock() + + if ip == nil { + next, err := getNextIp() + if err != nil { + return nil, err + } + return next, nil + } + + if err := validateIP(ip); err != nil { + return nil, err + } + return ip, nil +} + +func ReleaseIP(ip *net.IPAddr) error { + lock.Lock() + defer lock.Unlock() + +} + +func getNextIp(network iPNet) (net.IPAddr, error) { + if available, exists := availableIPS[network]; exists { + } + + var ( + netNetwork = newNetIPNet(network) + firstIP, _ = networkRange(netNetwork) + ipNum = ipToInt(firstIP) + ownIP = ipToInt(netNetwork.IP) + size = networkSize(netNetwork.Mask) + + pos = int32(1) + max = size - 2 // -1 for the broadcast address, -1 for the gateway address + ) + + for { + var ( + newNum int32 + inUse bool + ) + + // Find first unused IP, give up after one whole round + for attempt := int32(0); attempt < max; attempt++ { + newNum = ipNum + pos + + pos = pos%max + 1 + + // The network's IP is never okay to use + if newNum == ownIP { + continue + } + + if _, inUse = alloc.inUse[newNum]; !inUse { + // We found an unused IP + break + } + } + + ip := allocatedIP{ip: intToIP(newNum)} + if inUse { + ip.err = errors.New("No unallocated IP available") + } + + select { + case quit := <-alloc.quit: + if quit { + return + } + case alloc.queueAlloc <- ip: + alloc.inUse[newNum] = struct{}{} + case released := <-alloc.queueReleased: + r := ipToInt(released) + delete(alloc.inUse, r) + + if inUse { + // If we couldn't allocate a new IP, the released one + // will be the only free one now, so instantly use it + // next time + pos = r - ipNum + } else { + // Use same IP as last time + if pos == 1 { + pos = max + } else { + pos-- + } + } + } + } + +} + +func validateIP(ip *net.IPAddr) error { + +} + +func checkRouteOverlaps(networks []netlink.Route, toCheck *net.IPNet) error { + for _, network := range networks { + if network.IPNet != nil && networkOverlaps(toCheck, network.IPNet) { + return ErrNetworkAlreadyAllocated + } + } + return nil +} + +// Detects overlap between one IPNet and another +func networkOverlaps(netX *net.IPNet, netY *net.IPNet) bool { + if firstIP, _ := networkRange(netX); netY.Contains(firstIP) { + return true + } + if firstIP, _ := networkRange(netY); netX.Contains(firstIP) { + return true + } + return false +} + +func checkExistingNetworkOverlaps(network *net.IPNet) error { + for existing := range allocatedIPs { + if newIPNet(network) == existing { + return ErrNetworkAlreadyRegisterd + } + if networkOverlaps(network, existing) { + return ErrNetworkAlreadyAllocated + } + } + return nil +} + +// Calculates the first and last IP addresses in an IPNet +func networkRange(network *net.IPNet) (net.IP, net.IP) { + var ( + netIP = network.IP.To4() + firstIP = netIP.Mask(network.Mask) + lastIP = net.IPv4(0, 0, 0, 0).To4() + ) + + for i := 0; i < len(lastIP); i++ { + lastIP[i] = netIP[i] | ^network.Mask[i] + } + return firstIP, lastIP +} + +func newIPNet(network *net.IPNet) iPNet { + return iPNet{ + IP: string(network.IP), + Mask: string(network.Mask), + } +} + +func newNetIPNet(network iPNet) *net.IPNet { + return &net.IPNet{ + IP: []byte(network.IP), + Mask: []byte(network.Mask), + } +} + +// Converts a 4 bytes IP into a 32 bit integer +func ipToInt(ip net.IP) int32 { + return int32(binary.BigEndian.Uint32(ip.To4())) +} + +// Converts 32 bit integer into a 4 bytes IP address +func intToIP(n int32) net.IP { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(n)) + return net.IP(b) +} + +// Given a netmask, calculates the number of available hosts +func networkSize(mask net.IPMask) int32 { + m := net.IPv4Mask(0, 0, 0, 0) + for i := 0; i < net.IPv4len; i++ { + m[i] = ^mask[i] + } + + return int32(binary.BigEndian.Uint32(m)) + 1 +} From 63d18d875b8a136d59c98b7b571366b10b5d4f76 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 22 Jan 2014 16:00:53 -0800 Subject: [PATCH 02/12] Implement basic stack for ips Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) --- networkdriver/portallocator/allocator.go | 2 +- networkdriver/portallocator/ipset.go | 62 ++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 networkdriver/portallocator/ipset.go diff --git a/networkdriver/portallocator/allocator.go b/networkdriver/portallocator/allocator.go index 5d8e812309..3704fae485 100644 --- a/networkdriver/portallocator/allocator.go +++ b/networkdriver/portallocator/allocator.go @@ -5,11 +5,11 @@ import ( "errors" "github.com/dotcloud/docker/pkg/netlink" "net" + "sort" "sync" ) type networkSet map[iPNet]iPSet -type iPSet map[string]struct{} type iPNet struct { IP string diff --git a/networkdriver/portallocator/ipset.go b/networkdriver/portallocator/ipset.go new file mode 100644 index 0000000000..b7737bdb73 --- /dev/null +++ b/networkdriver/portallocator/ipset.go @@ -0,0 +1,62 @@ +package ipallocator + +import ( + "sync" +) + +// iPSet is a thread-safe sorted set and a stack. +type iPSet struct { + sync.RWMutex + set []string +} + +// Push takes a string and adds it to the set. If the elem aready exists, it has no effect. +func (s *iPSet) Push(elem string) { + s.RLock() + for i, e := range s.set { + if e == elem { + s.RUnlock() + return + } + } + s.RUnlock() + + s.Lock() + s.set = append(s.set, elem) + // Make sure the list is always sorted + sort.Strings(s.set) + s.Unlock() +} + +// Pop returns the first elemen from the list and removes it. +// If the list is empty, it returns an empty string +func (s *iPSet) Pop() string { + s.RLock() + + for i, e := range s.set { + ret := e + s.RUnlock() + s.Lock() + s.set = append(s.set[:i], s.set[i+1:]...) + s.Unlock() + return e + } + s.RUnlock() + return "" +} + +// Remove removes an element from the list. +// If the element is not found, it has no effect. +func (s *iPSet) Remove(elem string) { + for i, e := range s.set { + if e == elem { + s.set = append(s.set[:i], s.set[i+1:]...) + return + } + } +} + +// Len returns the length of the list. +func (s *iPSet) Len() int { + return len(s.set) +} From d32777f6b1f3a5dff6df0e285b9cd68b2c0d00ad Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 22 Jan 2014 16:13:22 -0800 Subject: [PATCH 03/12] Implement get next ip Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- networkdriver/portallocator/allocator.go | 105 +++++++++-------------- 1 file changed, 42 insertions(+), 63 deletions(-) diff --git a/networkdriver/portallocator/allocator.go b/networkdriver/portallocator/allocator.go index 5d8e812309..a148dc4138 100644 --- a/networkdriver/portallocator/allocator.go +++ b/networkdriver/portallocator/allocator.go @@ -19,6 +19,7 @@ type iPNet struct { var ( ErrNetworkAlreadyAllocated = errors.New("requested network overlaps with existing network") ErrNetworkAlreadyRegisterd = errors.New("requested network is already registered") + ErrNoAvailableIps = errors.New("no available ips on network") lock = sync.Mutex{} allocatedIPs = networkSet{} availableIPS = networkSet{} @@ -40,109 +41,87 @@ func RegisterNetwork(network *net.IPNet) error { if err := checkExistingNetworkOverlaps(network); err != nil { return err } - allocatedIPs[newIPNet(network)] = iPSet{} return nil } -func RequestIP(ip *net.IPAddr) (*net.IPAddr, error) { +func RequestIP(network *net.IPNet, ip *net.IPAddr) (*net.IPAddr, error) { lock.Lock() defer lock.Unlock() if ip == nil { - next, err := getNextIp() + next, err := getNextIp(network) if err != nil { return nil, err } return next, nil } - if err := validateIP(ip); err != nil { + if err := validateIP(network, ip); err != nil { return nil, err } return ip, nil } -func ReleaseIP(ip *net.IPAddr) error { +func ReleaseIP(network *net.IPNet, ip *net.IPAddr) error { lock.Lock() defer lock.Unlock() + n := newIPNet(network) + existing := allocatedIPs[n] + + delete(existing, ip.String()) + availableIPS[n][ip.String()] = struct{}{} + + return nil } -func getNextIp(network iPNet) (net.IPAddr, error) { +func getNextIp(network *net.IPNet) (net.IPAddr, error) { if available, exists := availableIPS[network]; exists { + return nil, nil } var ( - netNetwork = newNetIPNet(network) - firstIP, _ = networkRange(netNetwork) + firstIP, _ = networkRange(network) ipNum = ipToInt(firstIP) - ownIP = ipToInt(netNetwork.IP) - size = networkSize(netNetwork.Mask) + ownIP = ipToInt(network.IP) + size = networkSize(network.Mask) + n = newIPNet(network) + allocated = allocatedIPs[n] - pos = int32(1) - max = size - 2 // -1 for the broadcast address, -1 for the gateway address + pos = int32(1) + max = size - 2 // -1 for the broadcast address, -1 for the gateway address + ip *net.IP + newNum int32 + inUse bool ) - for { - var ( - newNum int32 - inUse bool - ) - - // Find first unused IP, give up after one whole round - for attempt := int32(0); attempt < max; attempt++ { - newNum = ipNum + pos - - pos = pos%max + 1 - - // The network's IP is never okay to use - if newNum == ownIP { - continue - } - - if _, inUse = alloc.inUse[newNum]; !inUse { - // We found an unused IP - break - } + // Find first unused IP, give up after one whole round + for attempt := int32(0); attempt < max; attempt++ { + newNum = ipNum + pos + pos = pos%max + 1 + // The network's IP is never okay to use + if newNum == ownIP { + continue } - ip := allocatedIP{ip: intToIP(newNum)} - if inUse { - ip.err = errors.New("No unallocated IP available") - } - - select { - case quit := <-alloc.quit: - if quit { - return - } - case alloc.queueAlloc <- ip: - alloc.inUse[newNum] = struct{}{} - case released := <-alloc.queueReleased: - r := ipToInt(released) - delete(alloc.inUse, r) - - if inUse { - // If we couldn't allocate a new IP, the released one - // will be the only free one now, so instantly use it - // next time - pos = r - ipNum - } else { - // Use same IP as last time - if pos == 1 { - pos = max - } else { - pos-- - } - } + ip = intToIP(newNum) + if _, inUse = allocated[ip.String()]; !inUse { + // We found an unused IP + break } } + if ip == nil { + return nil, ErrNoAvailableIps + } + allocated[ip.String()] = struct{}{} + + return ip, nil } -func validateIP(ip *net.IPAddr) error { +func validateIP(network *net.IPNet, ip *net.IPAddr) error { } From 43bcbf06a663c5d8cac63f2af8fefef7edc5513a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 22 Jan 2014 16:13:28 -0800 Subject: [PATCH 04/12] Implement Containers in set Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) --- networkdriver/portallocator/ipset.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/networkdriver/portallocator/ipset.go b/networkdriver/portallocator/ipset.go index b7737bdb73..83203a1688 100644 --- a/networkdriver/portallocator/ipset.go +++ b/networkdriver/portallocator/ipset.go @@ -45,6 +45,16 @@ func (s *iPSet) Pop() string { return "" } +// Exists checks if the given element present in the list. +func (s *iPSet) Exists(elem string) bool { + for _, e := range s.set { + if e == elem { + return true + } + } + return false +} + // Remove removes an element from the list. // If the element is not found, it has no effect. func (s *iPSet) Remove(elem string) { From 18df387bf8a4a1419b6d3267a4e0551ff72b0f88 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 22 Jan 2014 17:12:28 -0800 Subject: [PATCH 05/12] Implement PullBack() Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) --- networkdriver/portallocator/ipset.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/networkdriver/portallocator/ipset.go b/networkdriver/portallocator/ipset.go index 83203a1688..282279f29d 100644 --- a/networkdriver/portallocator/ipset.go +++ b/networkdriver/portallocator/ipset.go @@ -28,9 +28,14 @@ func (s *iPSet) Push(elem string) { s.Unlock() } +// Pop is an alias to PopFront() +func (s *iPSet) Pop() string { + return s.PopFront() +} + // Pop returns the first elemen from the list and removes it. // If the list is empty, it returns an empty string -func (s *iPSet) Pop() string { +func (s *iPSet) PopFront() string { s.RLock() for i, e := range s.set { @@ -45,6 +50,16 @@ func (s *iPSet) Pop() string { return "" } +// PullBack retrieve the last element of the list. +// The element is not removed. +// If the list is empty, an empty element is returned. +func (s *iPSet) PullBack() string { + if len(s.set) == 0 { + return "" + } + return s.set[len(s.set)-1] +} + // Exists checks if the given element present in the list. func (s *iPSet) Exists(elem string) bool { for _, e := range s.set { From 3e3abdd770bdc23c409a9e49619a1897ffbf2354 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 22 Jan 2014 17:20:19 -0800 Subject: [PATCH 06/12] Use int32 instead of string for ip set Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) --- networkdriver/portallocator/ipset.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/networkdriver/portallocator/ipset.go b/networkdriver/portallocator/ipset.go index 282279f29d..c54e64a120 100644 --- a/networkdriver/portallocator/ipset.go +++ b/networkdriver/portallocator/ipset.go @@ -1,17 +1,18 @@ package ipallocator import ( + "sort" "sync" ) // iPSet is a thread-safe sorted set and a stack. type iPSet struct { sync.RWMutex - set []string + set []int32 } // Push takes a string and adds it to the set. If the elem aready exists, it has no effect. -func (s *iPSet) Push(elem string) { +func (s *iPSet) Push(elem int32) { s.RLock() for i, e := range s.set { if e == elem { @@ -24,18 +25,18 @@ func (s *iPSet) Push(elem string) { s.Lock() s.set = append(s.set, elem) // Make sure the list is always sorted - sort.Strings(s.set) + sort.Ints(s.set) s.Unlock() } // Pop is an alias to PopFront() -func (s *iPSet) Pop() string { +func (s *iPSet) Pop() int32 { return s.PopFront() } // Pop returns the first elemen from the list and removes it. -// If the list is empty, it returns an empty string -func (s *iPSet) PopFront() string { +// If the list is empty, it returns 0 +func (s *iPSet) PopFront() int32 { s.RLock() for i, e := range s.set { @@ -53,7 +54,7 @@ func (s *iPSet) PopFront() string { // PullBack retrieve the last element of the list. // The element is not removed. // If the list is empty, an empty element is returned. -func (s *iPSet) PullBack() string { +func (s *iPSet) PullBack() int32 { if len(s.set) == 0 { return "" } @@ -61,7 +62,7 @@ func (s *iPSet) PullBack() string { } // Exists checks if the given element present in the list. -func (s *iPSet) Exists(elem string) bool { +func (s *iPSet) Exists(elem int32) bool { for _, e := range s.set { if e == elem { return true @@ -72,7 +73,7 @@ func (s *iPSet) Exists(elem string) bool { // Remove removes an element from the list. // If the element is not found, it has no effect. -func (s *iPSet) Remove(elem string) { +func (s *iPSet) Remove(elem int32) { for i, e := range s.set { if e == elem { s.set = append(s.set[:i], s.set[i+1:]...) From 6bc05899aa5023fdda0441b76a829d0e9a6f6dea Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 22 Jan 2014 18:05:20 -0800 Subject: [PATCH 07/12] Finish implementation and begin working on tests Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- networkdriver/portallocator/allocator.go | 101 +++++++++--------- networkdriver/portallocator/allocator_test.go | 26 +++++ networkdriver/portallocator/ipset.go | 23 ++-- 3 files changed, 89 insertions(+), 61 deletions(-) create mode 100644 networkdriver/portallocator/allocator_test.go diff --git a/networkdriver/portallocator/allocator.go b/networkdriver/portallocator/allocator.go index 40a1c001ce..91036de199 100644 --- a/networkdriver/portallocator/allocator.go +++ b/networkdriver/portallocator/allocator.go @@ -5,7 +5,6 @@ import ( "errors" "github.com/dotcloud/docker/pkg/netlink" "net" - "sort" "sync" ) @@ -20,9 +19,11 @@ var ( ErrNetworkAlreadyAllocated = errors.New("requested network overlaps with existing network") ErrNetworkAlreadyRegisterd = errors.New("requested network is already registered") ErrNoAvailableIps = errors.New("no available ips on network") - lock = sync.Mutex{} - allocatedIPs = networkSet{} - availableIPS = networkSet{} + ErrIPAlreadyAllocated = errors.New("ip already allocated") + + lock = sync.Mutex{} + allocatedIPs = networkSet{} + availableIPS = networkSet{} ) func RegisterNetwork(network *net.IPNet) error { @@ -41,12 +42,15 @@ func RegisterNetwork(network *net.IPNet) error { if err := checkExistingNetworkOverlaps(network); err != nil { return err } - allocatedIPs[newIPNet(network)] = iPSet{} + n := newIPNet(network) + + allocatedIPs[n] = iPSet{} + availableIPS[n] = iPSet{} return nil } -func RequestIP(network *net.IPNet, ip *net.IPAddr) (*net.IPAddr, error) { +func RequestIP(network *net.IPNet, ip *net.IP) (*net.IP, error) { lock.Lock() defer lock.Unlock() @@ -58,71 +62,65 @@ func RequestIP(network *net.IPNet, ip *net.IPAddr) (*net.IPAddr, error) { return next, nil } - if err := validateIP(network, ip); err != nil { + if err := registerIP(network, ip); err != nil { return nil, err } return ip, nil } -func ReleaseIP(network *net.IPNet, ip *net.IPAddr) error { +func ReleaseIP(network *net.IPNet, ip *net.IP) error { lock.Lock() defer lock.Unlock() n := newIPNet(network) existing := allocatedIPs[n] - delete(existing, ip.String()) - availableIPS[n][ip.String()] = struct{}{} + i := ipToInt(ip) + existing.Remove(int(i)) + available := availableIPS[n] + available.Push(int(i)) return nil } -func getNextIp(network *net.IPNet) (net.IPAddr, error) { - if available, exists := availableIPS[network]; exists { - return nil, nil - } - +func getNextIp(network *net.IPNet) (*net.IP, error) { var ( - firstIP, _ = networkRange(network) - ipNum = ipToInt(firstIP) - ownIP = ipToInt(network.IP) - size = networkSize(network.Mask) - n = newIPNet(network) - allocated = allocatedIPs[n] - - pos = int32(1) - max = size - 2 // -1 for the broadcast address, -1 for the gateway address - ip *net.IP - newNum int32 - inUse bool + n = newIPNet(network) + available = availableIPS[n] + next = available.Pop() + allocated = allocatedIPs[n] + ownIP = int(ipToInt(&network.IP)) ) - // Find first unused IP, give up after one whole round - for attempt := int32(0); attempt < max; attempt++ { - newNum = ipNum + pos - pos = pos%max + 1 - // The network's IP is never okay to use - if newNum == ownIP { + if next != 0 { + ip := intToIP(int32(next)) + allocated.Push(int(next)) + return ip, nil + } + size := int(networkSize(network.Mask)) + next = allocated.PullBack() + 1 + + // size -1 for the broadcast address, -1 for the gateway address + for i := 0; i < size-2; i++ { + if next == ownIP { + next++ continue } - ip = intToIP(newNum) - if _, inUse = allocated[ip.String()]; !inUse { - // We found an unused IP - break - } - } + ip := intToIP(int32(next)) + allocated.Push(next) - if ip == nil { - return nil, ErrNoAvailableIps + return ip, nil } - allocated[ip.String()] = struct{}{} - - return ip, nil + return nil, ErrNoAvailableIps } -func validateIP(network *net.IPNet, ip *net.IPAddr) error { - +func registerIP(network *net.IPNet, ip *net.IP) error { + existing := allocatedIPs[newIPNet(network)] + if existing.Exists(int(ipToInt(ip))) { + return ErrIPAlreadyAllocated + } + return nil } func checkRouteOverlaps(networks []netlink.Route, toCheck *net.IPNet) error { @@ -150,7 +148,9 @@ func checkExistingNetworkOverlaps(network *net.IPNet) error { if newIPNet(network) == existing { return ErrNetworkAlreadyRegisterd } - if networkOverlaps(network, existing) { + + ex := newNetIPNet(existing) + if networkOverlaps(network, ex) { return ErrNetworkAlreadyAllocated } } @@ -186,15 +186,16 @@ func newNetIPNet(network iPNet) *net.IPNet { } // Converts a 4 bytes IP into a 32 bit integer -func ipToInt(ip net.IP) int32 { +func ipToInt(ip *net.IP) int32 { return int32(binary.BigEndian.Uint32(ip.To4())) } // Converts 32 bit integer into a 4 bytes IP address -func intToIP(n int32) net.IP { +func intToIP(n int32) *net.IP { b := make([]byte, 4) binary.BigEndian.PutUint32(b, uint32(n)) - return net.IP(b) + ip := net.IP(b) + return &ip } // Given a netmask, calculates the number of available hosts diff --git a/networkdriver/portallocator/allocator_test.go b/networkdriver/portallocator/allocator_test.go new file mode 100644 index 0000000000..570f415780 --- /dev/null +++ b/networkdriver/portallocator/allocator_test.go @@ -0,0 +1,26 @@ +package ipallocator + +import ( + "net" + "testing" +) + +func TestRegisterNetwork(t *testing.T) { + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + if err := RegisterNetwork(network); err != nil { + t.Fatal(err) + } + + n := newIPNet(network) + if _, exists := allocatedIPs[n]; !exists { + t.Fatal("IPNet should exist in allocated IPs") + } + + if _, exists := availableIPS[n]; !exists { + t.Fatal("IPNet should exist in available IPs") + } +} diff --git a/networkdriver/portallocator/ipset.go b/networkdriver/portallocator/ipset.go index c54e64a120..42b545b2d7 100644 --- a/networkdriver/portallocator/ipset.go +++ b/networkdriver/portallocator/ipset.go @@ -8,13 +8,13 @@ import ( // iPSet is a thread-safe sorted set and a stack. type iPSet struct { sync.RWMutex - set []int32 + set []int } // Push takes a string and adds it to the set. If the elem aready exists, it has no effect. -func (s *iPSet) Push(elem int32) { +func (s *iPSet) Push(elem int) { s.RLock() - for i, e := range s.set { + for _, e := range s.set { if e == elem { s.RUnlock() return @@ -30,13 +30,13 @@ func (s *iPSet) Push(elem int32) { } // Pop is an alias to PopFront() -func (s *iPSet) Pop() int32 { +func (s *iPSet) Pop() int { return s.PopFront() } // Pop returns the first elemen from the list and removes it. // If the list is empty, it returns 0 -func (s *iPSet) PopFront() int32 { +func (s *iPSet) PopFront() int { s.RLock() for i, e := range s.set { @@ -45,24 +45,25 @@ func (s *iPSet) PopFront() int32 { s.Lock() s.set = append(s.set[:i], s.set[i+1:]...) s.Unlock() - return e + return ret } s.RUnlock() - return "" + + return 0 } // PullBack retrieve the last element of the list. // The element is not removed. // If the list is empty, an empty element is returned. -func (s *iPSet) PullBack() int32 { +func (s *iPSet) PullBack() int { if len(s.set) == 0 { - return "" + return 0 } return s.set[len(s.set)-1] } // Exists checks if the given element present in the list. -func (s *iPSet) Exists(elem int32) bool { +func (s *iPSet) Exists(elem int) bool { for _, e := range s.set { if e == elem { return true @@ -73,7 +74,7 @@ func (s *iPSet) Exists(elem int32) bool { // Remove removes an element from the list. // If the element is not found, it has no effect. -func (s *iPSet) Remove(elem int32) { +func (s *iPSet) Remove(elem int) { for i, e := range s.set { if e == elem { s.set = append(s.set[:i], s.set[i+1:]...) From 79bb8212e05cc9c14b8edda2b8a924fef63ea2e0 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 22 Jan 2014 19:34:47 -0800 Subject: [PATCH 08/12] Implement pos for set and add unit tests Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- networkdriver/portallocator/allocator.go | 62 +++++--- networkdriver/portallocator/allocator_test.go | 137 ++++++++++++++++++ networkdriver/portallocator/ipset.go | 5 - 3 files changed, 178 insertions(+), 26 deletions(-) diff --git a/networkdriver/portallocator/allocator.go b/networkdriver/portallocator/allocator.go index 91036de199..5b6befd1da 100644 --- a/networkdriver/portallocator/allocator.go +++ b/networkdriver/portallocator/allocator.go @@ -8,7 +8,7 @@ import ( "sync" ) -type networkSet map[iPNet]iPSet +type networkSet map[iPNet]*iPSet type iPNet struct { IP string @@ -44,8 +44,8 @@ func RegisterNetwork(network *net.IPNet) error { } n := newIPNet(network) - allocatedIPs[n] = iPSet{} - availableIPS[n] = iPSet{} + allocatedIPs[n] = &iPSet{} + availableIPS[n] = &iPSet{} return nil } @@ -72,13 +72,18 @@ func ReleaseIP(network *net.IPNet, ip *net.IP) error { lock.Lock() defer lock.Unlock() - n := newIPNet(network) - existing := allocatedIPs[n] + var ( + first, _ = networkRange(network) + base = ipToInt(&first) + n = newIPNet(network) + existing = allocatedIPs[n] + available = availableIPS[n] + i = ipToInt(ip) + pos = i - base + ) - i := ipToInt(ip) - existing.Remove(int(i)) - available := availableIPS[n] - available.Push(int(i)) + existing.Remove(int(pos)) + available.Push(int(pos)) return nil } @@ -86,29 +91,43 @@ func ReleaseIP(network *net.IPNet, ip *net.IP) error { func getNextIp(network *net.IPNet) (*net.IP, error) { var ( n = newIPNet(network) + ownIP = ipToInt(&network.IP) available = availableIPS[n] - next = available.Pop() allocated = allocatedIPs[n] - ownIP = int(ipToInt(&network.IP)) + + first, _ = networkRange(network) + base = ipToInt(&first) + + pos = int32(available.Pop()) ) - if next != 0 { - ip := intToIP(int32(next)) - allocated.Push(int(next)) + // We pop and push the position not the ip + if pos != 0 { + ip := intToIP(int32(base + pos)) + allocated.Push(int(pos)) + return ip, nil } - size := int(networkSize(network.Mask)) - next = allocated.PullBack() + 1 - // size -1 for the broadcast address, -1 for the gateway address - for i := 0; i < size-2; i++ { + var ( + size = int(networkSize(network.Mask)) + max = int32(size - 2) // size -1 for the broadcast address, -1 for the gateway address + ) + + if pos = int32(allocated.PullBack()); pos == 0 { + pos = 1 + } + + for i := int32(0); i < max; i++ { + next := int32(base + pos) + pos = pos%max + 1 + if next == ownIP { - next++ continue } - ip := intToIP(int32(next)) - allocated.Push(next) + ip := intToIP(next) + allocated.Push(int(pos)) return ip, nil } @@ -117,6 +136,7 @@ func getNextIp(network *net.IPNet) (*net.IP, error) { func registerIP(network *net.IPNet, ip *net.IP) error { existing := allocatedIPs[newIPNet(network)] + // checking position not ip if existing.Exists(int(ipToInt(ip))) { return ErrIPAlreadyAllocated } diff --git a/networkdriver/portallocator/allocator_test.go b/networkdriver/portallocator/allocator_test.go index 570f415780..bcdcfa66b8 100644 --- a/networkdriver/portallocator/allocator_test.go +++ b/networkdriver/portallocator/allocator_test.go @@ -1,11 +1,18 @@ package ipallocator import ( + "fmt" "net" "testing" ) +func reset() { + allocatedIPs = networkSet{} + availableIPS = networkSet{} +} + func TestRegisterNetwork(t *testing.T) { + defer reset() network := &net.IPNet{ IP: []byte{192, 168, 0, 1}, Mask: []byte{255, 255, 255, 0}, @@ -24,3 +31,133 @@ func TestRegisterNetwork(t *testing.T) { t.Fatal("IPNet should exist in available IPs") } } + +func TestRegisterTwoNetworks(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + if err := RegisterNetwork(network); err != nil { + t.Fatal(err) + } + + network2 := &net.IPNet{ + IP: []byte{10, 1, 42, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + if err := RegisterNetwork(network2); err != nil { + t.Fatal(err) + } +} + +func TestRegisterNetworkThatExists(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + if err := RegisterNetwork(network); err != nil { + t.Fatal(err) + } + + if err := RegisterNetwork(network); err != ErrNetworkAlreadyRegisterd { + t.Fatalf("Expected error of %s got %s", ErrNetworkAlreadyRegisterd, err) + } +} + +func TestRequestNewIps(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + if err := RegisterNetwork(network); err != nil { + t.Fatal(err) + } + + for i := 2; i < 10; i++ { + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + if expected := fmt.Sprintf("192.168.0.%d", i); ip.String() != expected { + t.Fatalf("Expected ip %s got %s", expected, ip.String()) + } + } +} + +func TestReleaseIp(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + if err := RegisterNetwork(network); err != nil { + t.Fatal(err) + } + + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + if err := ReleaseIP(network, ip); err != nil { + t.Fatal(err) + } +} + +func TestGetReleasedIp(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + if err := RegisterNetwork(network); err != nil { + t.Fatal(err) + } + + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + value := ip.String() + if err := ReleaseIP(network, ip); err != nil { + t.Fatal(err) + } + + ip, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + if ip.String() != value { + t.Fatalf("Expected to receive same ip %s got %s", value, ip.String()) + } +} + +func TestRequesetSpecificIp(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + if err := RegisterNetwork(network); err != nil { + t.Fatal(err) + } + + ip := net.ParseIP("192.168.1.5") + + if _, err := RequestIP(network, &ip); err != nil { + t.Fatal(err) + } +} diff --git a/networkdriver/portallocator/ipset.go b/networkdriver/portallocator/ipset.go index 42b545b2d7..43d54691d1 100644 --- a/networkdriver/portallocator/ipset.go +++ b/networkdriver/portallocator/ipset.go @@ -82,8 +82,3 @@ func (s *iPSet) Remove(elem int) { } } } - -// Len returns the length of the list. -func (s *iPSet) Len() int { - return len(s.set) -} From 648357ffdf6670810383be6437b8a5f8a38d4e42 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 23 Jan 2014 00:28:24 -0800 Subject: [PATCH 09/12] Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- networkdriver/{portallocator => ipallocator}/allocator.go | 0 networkdriver/{portallocator => ipallocator}/allocator_test.go | 0 networkdriver/{portallocator => ipallocator}/ipset.go | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename networkdriver/{portallocator => ipallocator}/allocator.go (100%) rename networkdriver/{portallocator => ipallocator}/allocator_test.go (100%) rename networkdriver/{portallocator => ipallocator}/ipset.go (100%) diff --git a/networkdriver/portallocator/allocator.go b/networkdriver/ipallocator/allocator.go similarity index 100% rename from networkdriver/portallocator/allocator.go rename to networkdriver/ipallocator/allocator.go diff --git a/networkdriver/portallocator/allocator_test.go b/networkdriver/ipallocator/allocator_test.go similarity index 100% rename from networkdriver/portallocator/allocator_test.go rename to networkdriver/ipallocator/allocator_test.go diff --git a/networkdriver/portallocator/ipset.go b/networkdriver/ipallocator/ipset.go similarity index 100% rename from networkdriver/portallocator/ipset.go rename to networkdriver/ipallocator/ipset.go From fccca3542b91513dd0da73db21ffa3ced4dd9b7f Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 23 Jan 2014 01:31:38 -0800 Subject: [PATCH 10/12] Move tests from core into ipallocator Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- container.go | 6 +- network.go | 115 ++----- network_test.go | 273 ---------------- networkdriver/ipallocator/allocator.go | 33 +- networkdriver/ipallocator/allocator_test.go | 329 +++++++++++++++++++- networkdriver/network.go | 1 + 6 files changed, 378 insertions(+), 379 deletions(-) create mode 100644 networkdriver/network.go diff --git a/container.go b/container.go index 7ebfb3e397..f7bcec99d8 100644 --- a/container.go +++ b/container.go @@ -7,6 +7,7 @@ import ( "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/execdriver" "github.com/dotcloud/docker/graphdriver" + "github.com/dotcloud/docker/networkdriver/ipallocator" "github.com/dotcloud/docker/pkg/mount" "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/utils" @@ -1039,8 +1040,9 @@ func (container *Container) allocateNetwork() error { manager: manager, } if iface != nil && iface.IPNet.IP != nil { - ipNum := ipToInt(iface.IPNet.IP) - manager.ipAllocator.inUse[ipNum] = struct{}{} + if _, err := ipallocator.RequestIP(manager.bridgeNetwork, &iface.IPNet.IP); err != nil { + return err + } } else { iface, err = container.runtime.networkManager.Allocate() if err != nil { diff --git a/network.go b/network.go index 473df0ffdb..4987140402 100644 --- a/network.go +++ b/network.go @@ -1,8 +1,8 @@ package docker import ( - "errors" "fmt" + "github.com/dotcloud/docker/networkdriver/ipallocator" "github.com/dotcloud/docker/pkg/iptables" "github.com/dotcloud/docker/pkg/netlink" "github.com/dotcloud/docker/proxy" @@ -24,21 +24,6 @@ const ( siocBRADDBR = 0x89a0 ) -func checkNameserverOverlaps(nameservers []string, dockerNetwork *net.IPNet) error { - if len(nameservers) > 0 { - for _, ns := range nameservers { - _, nsNetwork, err := net.ParseCIDR(ns) - if err != nil { - return err - } - if networkOverlaps(dockerNetwork, nsNetwork) { - return fmt.Errorf("%s overlaps nameserver %s", dockerNetwork, nsNetwork) - } - } - } - return nil -} - // CreateBridgeIface creates a network bridge interface on the host system with the name `ifaceName`, // and attempts to configure it with an address which doesn't conflict with any other interface on the host. // If it can't find an address which doesn't conflict, it will return an error. @@ -86,17 +71,16 @@ func CreateBridgeIface(config *DaemonConfig) error { if err != nil { return err } - // TODO: @crosbymichael register route - if err := checkRouteOverlaps(routes, dockerNetwork); err == nil { - if err := checkNameserverOverlaps(nameservers, dockerNetwork); err == nil { - ifaceAddr = addr - break - } + + if err := ipallocator.RegisterNetwork(dockerNetwork, nameservers); err == nil { + ifaceAddr = addr + break } else { utils.Debugf("%s: %s", addr, err) } } } + if ifaceAddr == "" { return fmt.Errorf("Could not find a free IP address range for interface '%s'. Please configure its address manually and run 'docker -b %s'", config.BridgeIface, config.BridgeIface) } @@ -367,54 +351,6 @@ func newPortAllocator() (*PortAllocator, error) { return allocator, nil } -// IP allocator: Automatically allocate and release networking ports -type IPAllocator struct { - network *net.IPNet - queueAlloc chan allocatedIP - queueReleased chan net.IP - inUse map[int32]struct{} - quit chan bool -} - -type allocatedIP struct { - ip net.IP - err error -} - -func (alloc *IPAllocator) run() { -} - -func (alloc *IPAllocator) Acquire() (net.IP, error) { - ip := <-alloc.queueAlloc - return ip.ip, ip.err -} - -func (alloc *IPAllocator) Release(ip net.IP) { - alloc.queueReleased <- ip -} - -func (alloc *IPAllocator) Close() error { - alloc.quit <- true - close(alloc.quit) - close(alloc.queueAlloc) - close(alloc.queueReleased) - return nil -} - -func newIPAllocator(network *net.IPNet) *IPAllocator { - alloc := &IPAllocator{ - network: network, - queueAlloc: make(chan allocatedIP), - queueReleased: make(chan net.IP), - inUse: make(map[int32]struct{}), - quit: make(chan bool), - } - - go alloc.run() - - return alloc -} - // Network interface represents the networking stack of a container type NetworkInterface struct { IPNet net.IPNet @@ -519,7 +455,9 @@ func (iface *NetworkInterface) Release() { } } - iface.manager.ipAllocator.Release(iface.IPNet.IP) + if err := ipallocator.ReleaseIP(iface.manager.bridgeNetwork, &iface.IPNet.IP); err != nil { + log.Printf("Unable to release ip %s\n", err) + } } // Network Manager manages a set of network interfaces @@ -528,7 +466,6 @@ type NetworkManager struct { bridgeIface string bridgeNetwork *net.IPNet - ipAllocator *IPAllocator tcpPortAllocator *PortAllocator udpPortAllocator *PortAllocator portMapper *PortMapper @@ -543,27 +480,31 @@ func (manager *NetworkManager) Allocate() (*NetworkInterface, error) { return &NetworkInterface{disabled: true}, nil } - var ip net.IP + var ip *net.IP var err error - ip, err = manager.ipAllocator.Acquire() + ip, err = ipallocator.RequestIP(manager.bridgeNetwork, nil) if err != nil { return nil, err } - // avoid duplicate IP - ipNum := ipToInt(ip) - firstIP := manager.ipAllocator.network.IP.To4().Mask(manager.ipAllocator.network.Mask) - firstIPNum := ipToInt(firstIP) + 1 - if firstIPNum == ipNum { - ip, err = manager.ipAllocator.Acquire() - if err != nil { - return nil, err + // TODO: @crosbymichael why are we doing this ? + /* + // avoid duplicate IP + ipNum := ipToInt(ip) + firstIP := manager.ipAllocator.network.IP.To4().Mask(manager.ipAllocator.network.Mask) + firstIPNum := ipToInt(firstIP) + 1 + + if firstIPNum == ipNum { + ip, err = manager.ipAllocator.Acquire() + if err != nil { + return nil, err + } } - } + */ iface := &NetworkInterface{ - IPNet: net.IPNet{IP: ip, Mask: manager.bridgeNetwork.Mask}, + IPNet: net.IPNet{IP: *ip, Mask: manager.bridgeNetwork.Mask}, Gateway: manager.bridgeNetwork.IP, manager: manager, } @@ -576,14 +517,13 @@ func (manager *NetworkManager) Close() error { } err1 := manager.tcpPortAllocator.Close() err2 := manager.udpPortAllocator.Close() - err3 := manager.ipAllocator.Close() if err1 != nil { return err1 } if err2 != nil { return err2 } - return err3 + return nil } func newNetworkManager(config *DaemonConfig) (*NetworkManager, error) { @@ -670,8 +610,6 @@ func newNetworkManager(config *DaemonConfig) (*NetworkManager, error) { } } - ipAllocator := newIPAllocator(network) - tcpPortAllocator, err := newPortAllocator() if err != nil { return nil, err @@ -690,7 +628,6 @@ func newNetworkManager(config *DaemonConfig) (*NetworkManager, error) { manager := &NetworkManager{ bridgeIface: config.BridgeIface, bridgeNetwork: network, - ipAllocator: ipAllocator, tcpPortAllocator: tcpPortAllocator, udpPortAllocator: udpPortAllocator, portMapper: portMapper, diff --git a/network_test.go b/network_test.go index 0b6857ba76..0d25ccb158 100644 --- a/network_test.go +++ b/network_test.go @@ -2,9 +2,7 @@ package docker import ( "github.com/dotcloud/docker/pkg/iptables" - "github.com/dotcloud/docker/pkg/netlink" "github.com/dotcloud/docker/proxy" - "net" "testing" ) @@ -53,277 +51,6 @@ func TestPortAllocation(t *testing.T) { } } -func TestNetworkRange(t *testing.T) { - // Simple class C test - _, network, _ := net.ParseCIDR("192.168.0.1/24") - first, last := networkRange(network) - if !first.Equal(net.ParseIP("192.168.0.0")) { - t.Error(first.String()) - } - if !last.Equal(net.ParseIP("192.168.0.255")) { - t.Error(last.String()) - } - if size := networkSize(network.Mask); size != 256 { - t.Error(size) - } - - // Class A test - _, network, _ = net.ParseCIDR("10.0.0.1/8") - first, last = networkRange(network) - if !first.Equal(net.ParseIP("10.0.0.0")) { - t.Error(first.String()) - } - if !last.Equal(net.ParseIP("10.255.255.255")) { - t.Error(last.String()) - } - if size := networkSize(network.Mask); size != 16777216 { - t.Error(size) - } - - // Class A, random IP address - _, network, _ = net.ParseCIDR("10.1.2.3/8") - first, last = networkRange(network) - if !first.Equal(net.ParseIP("10.0.0.0")) { - t.Error(first.String()) - } - if !last.Equal(net.ParseIP("10.255.255.255")) { - t.Error(last.String()) - } - - // 32bit mask - _, network, _ = net.ParseCIDR("10.1.2.3/32") - first, last = networkRange(network) - if !first.Equal(net.ParseIP("10.1.2.3")) { - t.Error(first.String()) - } - if !last.Equal(net.ParseIP("10.1.2.3")) { - t.Error(last.String()) - } - if size := networkSize(network.Mask); size != 1 { - t.Error(size) - } - - // 31bit mask - _, network, _ = net.ParseCIDR("10.1.2.3/31") - first, last = networkRange(network) - if !first.Equal(net.ParseIP("10.1.2.2")) { - t.Error(first.String()) - } - if !last.Equal(net.ParseIP("10.1.2.3")) { - t.Error(last.String()) - } - if size := networkSize(network.Mask); size != 2 { - t.Error(size) - } - - // 26bit mask - _, network, _ = net.ParseCIDR("10.1.2.3/26") - first, last = networkRange(network) - if !first.Equal(net.ParseIP("10.1.2.0")) { - t.Error(first.String()) - } - if !last.Equal(net.ParseIP("10.1.2.63")) { - t.Error(last.String()) - } - if size := networkSize(network.Mask); size != 64 { - t.Error(size) - } -} - -func TestConversion(t *testing.T) { - ip := net.ParseIP("127.0.0.1") - i := ipToInt(ip) - if i == 0 { - t.Fatal("converted to zero") - } - conv := intToIP(i) - if !ip.Equal(conv) { - t.Error(conv.String()) - } -} - -func TestIPAllocator(t *testing.T) { - expectedIPs := []net.IP{ - 0: net.IPv4(127, 0, 0, 2), - 1: net.IPv4(127, 0, 0, 3), - 2: net.IPv4(127, 0, 0, 4), - 3: net.IPv4(127, 0, 0, 5), - 4: net.IPv4(127, 0, 0, 6), - } - - gwIP, n, _ := net.ParseCIDR("127.0.0.1/29") - alloc := newIPAllocator(&net.IPNet{IP: gwIP, Mask: n.Mask}) - // Pool after initialisation (f = free, u = used) - // 2(f) - 3(f) - 4(f) - 5(f) - 6(f) - // ↑ - - // Check that we get 5 IPs, from 127.0.0.2–127.0.0.6, in that - // order. - for i := 0; i < 5; i++ { - ip, err := alloc.Acquire() - if err != nil { - t.Fatal(err) - } - - assertIPEquals(t, expectedIPs[i], ip) - } - // Before loop begin - // 2(f) - 3(f) - 4(f) - 5(f) - 6(f) - // ↑ - - // After i = 0 - // 2(u) - 3(f) - 4(f) - 5(f) - 6(f) - // ↑ - - // After i = 1 - // 2(u) - 3(u) - 4(f) - 5(f) - 6(f) - // ↑ - - // After i = 2 - // 2(u) - 3(u) - 4(u) - 5(f) - 6(f) - // ↑ - - // After i = 3 - // 2(u) - 3(u) - 4(u) - 5(u) - 6(f) - // ↑ - - // After i = 4 - // 2(u) - 3(u) - 4(u) - 5(u) - 6(u) - // ↑ - - // Check that there are no more IPs - _, err := alloc.Acquire() - if err == nil { - t.Fatal("There shouldn't be any IP addresses at this point") - } - - // Release some IPs in non-sequential order - alloc.Release(expectedIPs[3]) - // 2(u) - 3(u) - 4(u) - 5(f) - 6(u) - // ↑ - - alloc.Release(expectedIPs[2]) - // 2(u) - 3(u) - 4(f) - 5(f) - 6(u) - // ↑ - - alloc.Release(expectedIPs[4]) - // 2(u) - 3(u) - 4(f) - 5(f) - 6(f) - // ↑ - - // Make sure that IPs are reused in sequential order, starting - // with the first released IP - newIPs := make([]net.IP, 3) - for i := 0; i < 3; i++ { - ip, err := alloc.Acquire() - if err != nil { - t.Fatal(err) - } - - newIPs[i] = ip - } - // Before loop begin - // 2(u) - 3(u) - 4(f) - 5(f) - 6(f) - // ↑ - - // After i = 0 - // 2(u) - 3(u) - 4(f) - 5(u) - 6(f) - // ↑ - - // After i = 1 - // 2(u) - 3(u) - 4(f) - 5(u) - 6(u) - // ↑ - - // After i = 2 - // 2(u) - 3(u) - 4(u) - 5(u) - 6(u) - // ↑ - - assertIPEquals(t, expectedIPs[3], newIPs[0]) - assertIPEquals(t, expectedIPs[4], newIPs[1]) - assertIPEquals(t, expectedIPs[2], newIPs[2]) - - _, err = alloc.Acquire() - if err == nil { - t.Fatal("There shouldn't be any IP addresses at this point") - } -} - -func assertIPEquals(t *testing.T, ip1, ip2 net.IP) { - if !ip1.Equal(ip2) { - t.Fatalf("Expected IP %s, got %s", ip1, ip2) - } -} - -func AssertOverlap(CIDRx string, CIDRy string, t *testing.T) { - _, netX, _ := net.ParseCIDR(CIDRx) - _, netY, _ := net.ParseCIDR(CIDRy) - if !networkOverlaps(netX, netY) { - t.Errorf("%v and %v should overlap", netX, netY) - } -} - -func AssertNoOverlap(CIDRx string, CIDRy string, t *testing.T) { - _, netX, _ := net.ParseCIDR(CIDRx) - _, netY, _ := net.ParseCIDR(CIDRy) - if networkOverlaps(netX, netY) { - t.Errorf("%v and %v should not overlap", netX, netY) - } -} - -func TestNetworkOverlaps(t *testing.T) { - //netY starts at same IP and ends within netX - AssertOverlap("172.16.0.1/24", "172.16.0.1/25", t) - //netY starts within netX and ends at same IP - AssertOverlap("172.16.0.1/24", "172.16.0.128/25", t) - //netY starts and ends within netX - AssertOverlap("172.16.0.1/24", "172.16.0.64/25", t) - //netY starts at same IP and ends outside of netX - AssertOverlap("172.16.0.1/24", "172.16.0.1/23", t) - //netY starts before and ends at same IP of netX - AssertOverlap("172.16.1.1/24", "172.16.0.1/23", t) - //netY starts before and ends outside of netX - AssertOverlap("172.16.1.1/24", "172.16.0.1/23", t) - //netY starts and ends before netX - AssertNoOverlap("172.16.1.1/25", "172.16.0.1/24", t) - //netX starts and ends before netY - AssertNoOverlap("172.16.1.1/25", "172.16.2.1/24", t) -} - -func TestCheckRouteOverlaps(t *testing.T) { - routesData := []string{"10.0.2.0/32", "10.0.3.0/24", "10.0.42.0/24", "172.16.42.0/24", "192.168.142.0/24"} - - routes := []netlink.Route{} - for _, addr := range routesData { - _, netX, _ := net.ParseCIDR(addr) - routes = append(routes, netlink.Route{IPNet: netX}) - } - - _, netX, _ := net.ParseCIDR("172.16.0.1/24") - if err := checkRouteOverlaps(routes, netX); err != nil { - t.Fatal(err) - } - - _, netX, _ = net.ParseCIDR("10.0.2.0/24") - if err := checkRouteOverlaps(routes, netX); err == nil { - t.Fatalf("10.0.2.0/24 and 10.0.2.0 should overlap but it doesn't") - } -} - -func TestCheckNameserverOverlaps(t *testing.T) { - nameservers := []string{"10.0.2.3/32", "192.168.102.1/32"} - - _, netX, _ := net.ParseCIDR("10.0.2.3/32") - - if err := checkNameserverOverlaps(nameservers, netX); err == nil { - t.Fatalf("%s should overlap 10.0.2.3/32 but doesn't", netX) - } - - _, netX, _ = net.ParseCIDR("192.168.102.2/32") - - if err := checkNameserverOverlaps(nameservers, netX); err != nil { - t.Fatalf("%s should not overlap %v but it does", netX, nameservers) - } -} - type StubProxy struct { frontendAddr *net.Addr backendAddr *net.Addr diff --git a/networkdriver/ipallocator/allocator.go b/networkdriver/ipallocator/allocator.go index 5b6befd1da..e6d39446df 100644 --- a/networkdriver/ipallocator/allocator.go +++ b/networkdriver/ipallocator/allocator.go @@ -16,20 +16,25 @@ type iPNet struct { } var ( - ErrNetworkAlreadyAllocated = errors.New("requested network overlaps with existing network") - ErrNetworkAlreadyRegisterd = errors.New("requested network is already registered") - ErrNoAvailableIps = errors.New("no available ips on network") - ErrIPAlreadyAllocated = errors.New("ip already allocated") + ErrNetworkAlreadyAllocated = errors.New("requested network overlaps with existing network") + ErrNetworkAlreadyRegisterd = errors.New("requested network is already registered") + ErrNetworkOverlapsWithNameservers = errors.New("requested network overlaps with nameserver") + ErrNoAvailableIps = errors.New("no available ips on network") + ErrIPAlreadyAllocated = errors.New("ip already allocated") lock = sync.Mutex{} allocatedIPs = networkSet{} availableIPS = networkSet{} ) -func RegisterNetwork(network *net.IPNet) error { +func RegisterNetwork(network *net.IPNet, nameservers []string) error { lock.Lock() defer lock.Unlock() + if err := checkExistingNetworkOverlaps(network); err != nil { + return err + } + routes, err := netlink.NetworkGetRoutes() if err != nil { return err @@ -39,9 +44,10 @@ func RegisterNetwork(network *net.IPNet) error { return err } - if err := checkExistingNetworkOverlaps(network); err != nil { + if err := checkNameserverOverlaps(nameservers, network); err != nil { return err } + n := newIPNet(network) allocatedIPs[n] = &iPSet{} @@ -227,3 +233,18 @@ func networkSize(mask net.IPMask) int32 { return int32(binary.BigEndian.Uint32(m)) + 1 } + +func checkNameserverOverlaps(nameservers []string, toCheck *net.IPNet) error { + if len(nameservers) > 0 { + for _, ns := range nameservers { + _, nsNetwork, err := net.ParseCIDR(ns) + if err != nil { + return err + } + if networkOverlaps(toCheck, nsNetwork) { + return ErrNetworkOverlapsWithNameservers + } + } + } + return nil +} diff --git a/networkdriver/ipallocator/allocator_test.go b/networkdriver/ipallocator/allocator_test.go index bcdcfa66b8..d1a58a021d 100644 --- a/networkdriver/ipallocator/allocator_test.go +++ b/networkdriver/ipallocator/allocator_test.go @@ -2,6 +2,7 @@ package ipallocator import ( "fmt" + "github.com/dotcloud/docker/pkg/netlink" "net" "testing" ) @@ -18,7 +19,7 @@ func TestRegisterNetwork(t *testing.T) { Mask: []byte{255, 255, 255, 0}, } - if err := RegisterNetwork(network); err != nil { + if err := RegisterNetwork(network, nil); err != nil { t.Fatal(err) } @@ -39,7 +40,7 @@ func TestRegisterTwoNetworks(t *testing.T) { Mask: []byte{255, 255, 255, 0}, } - if err := RegisterNetwork(network); err != nil { + if err := RegisterNetwork(network, nil); err != nil { t.Fatal(err) } @@ -48,7 +49,7 @@ func TestRegisterTwoNetworks(t *testing.T) { Mask: []byte{255, 255, 255, 0}, } - if err := RegisterNetwork(network2); err != nil { + if err := RegisterNetwork(network2, nil); err != nil { t.Fatal(err) } } @@ -60,11 +61,11 @@ func TestRegisterNetworkThatExists(t *testing.T) { Mask: []byte{255, 255, 255, 0}, } - if err := RegisterNetwork(network); err != nil { + if err := RegisterNetwork(network, nil); err != nil { t.Fatal(err) } - if err := RegisterNetwork(network); err != ErrNetworkAlreadyRegisterd { + if err := RegisterNetwork(network, nil); err != ErrNetworkAlreadyRegisterd { t.Fatalf("Expected error of %s got %s", ErrNetworkAlreadyRegisterd, err) } } @@ -76,7 +77,7 @@ func TestRequestNewIps(t *testing.T) { Mask: []byte{255, 255, 255, 0}, } - if err := RegisterNetwork(network); err != nil { + if err := RegisterNetwork(network, nil); err != nil { t.Fatal(err) } @@ -99,7 +100,7 @@ func TestReleaseIp(t *testing.T) { Mask: []byte{255, 255, 255, 0}, } - if err := RegisterNetwork(network); err != nil { + if err := RegisterNetwork(network, nil); err != nil { t.Fatal(err) } @@ -120,7 +121,7 @@ func TestGetReleasedIp(t *testing.T) { Mask: []byte{255, 255, 255, 0}, } - if err := RegisterNetwork(network); err != nil { + if err := RegisterNetwork(network, nil); err != nil { t.Fatal(err) } @@ -151,7 +152,7 @@ func TestRequesetSpecificIp(t *testing.T) { Mask: []byte{255, 255, 255, 0}, } - if err := RegisterNetwork(network); err != nil { + if err := RegisterNetwork(network, nil); err != nil { t.Fatal(err) } @@ -161,3 +162,313 @@ func TestRequesetSpecificIp(t *testing.T) { t.Fatal(err) } } + +func TestNonOverlapingNameservers(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + nameservers := []string{ + "127.0.0.1/32", + } + + if err := RegisterNetwork(network, nameservers); err != nil { + t.Fatal(err) + } +} + +func TestOverlapingNameservers(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + nameservers := []string{ + "192.168.0.1/32", + } + + if err := RegisterNetwork(network, nameservers); err != ErrNetworkOverlapsWithNameservers { + t.Fatalf("Expectecd error of %s got %s", ErrNetworkOverlapsWithNameservers, err) + } +} + +func TestNetworkRange(t *testing.T) { + // Simple class C test + _, network, _ := net.ParseCIDR("192.168.0.1/24") + first, last := networkRange(network) + if !first.Equal(net.ParseIP("192.168.0.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("192.168.0.255")) { + t.Error(last.String()) + } + if size := networkSize(network.Mask); size != 256 { + t.Error(size) + } + + // Class A test + _, network, _ = net.ParseCIDR("10.0.0.1/8") + first, last = networkRange(network) + if !first.Equal(net.ParseIP("10.0.0.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.255.255.255")) { + t.Error(last.String()) + } + if size := networkSize(network.Mask); size != 16777216 { + t.Error(size) + } + + // Class A, random IP address + _, network, _ = net.ParseCIDR("10.1.2.3/8") + first, last = networkRange(network) + if !first.Equal(net.ParseIP("10.0.0.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.255.255.255")) { + t.Error(last.String()) + } + + // 32bit mask + _, network, _ = net.ParseCIDR("10.1.2.3/32") + first, last = networkRange(network) + if !first.Equal(net.ParseIP("10.1.2.3")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.1.2.3")) { + t.Error(last.String()) + } + if size := networkSize(network.Mask); size != 1 { + t.Error(size) + } + + // 31bit mask + _, network, _ = net.ParseCIDR("10.1.2.3/31") + first, last = networkRange(network) + if !first.Equal(net.ParseIP("10.1.2.2")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.1.2.3")) { + t.Error(last.String()) + } + if size := networkSize(network.Mask); size != 2 { + t.Error(size) + } + + // 26bit mask + _, network, _ = net.ParseCIDR("10.1.2.3/26") + first, last = networkRange(network) + if !first.Equal(net.ParseIP("10.1.2.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.1.2.63")) { + t.Error(last.String()) + } + if size := networkSize(network.Mask); size != 64 { + t.Error(size) + } +} + +func TestConversion(t *testing.T) { + ip := net.ParseIP("127.0.0.1") + i := ipToInt(&ip) + if i == 0 { + t.Fatal("converted to zero") + } + conv := intToIP(i) + if !ip.Equal(*conv) { + t.Error(conv.String()) + } +} + +func TestIPAllocator(t *testing.T) { + expectedIPs := []net.IP{ + 0: net.IPv4(127, 0, 0, 2), + 1: net.IPv4(127, 0, 0, 3), + 2: net.IPv4(127, 0, 0, 4), + 3: net.IPv4(127, 0, 0, 5), + 4: net.IPv4(127, 0, 0, 6), + } + + gwIP, n, _ := net.ParseCIDR("127.0.0.1/29") + network := &net.IPNet{IP: gwIP, Mask: n.Mask} + if err := RegisterNetwork(network, nil); err != nil { + t.Fatal(err) + } + // Pool after initialisation (f = free, u = used) + // 2(f) - 3(f) - 4(f) - 5(f) - 6(f) + // ↑ + + // Check that we get 5 IPs, from 127.0.0.2–127.0.0.6, in that + // order. + for i := 0; i < 5; i++ { + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + assertIPEquals(t, &expectedIPs[i], ip) + } + // Before loop begin + // 2(f) - 3(f) - 4(f) - 5(f) - 6(f) + // ↑ + + // After i = 0 + // 2(u) - 3(f) - 4(f) - 5(f) - 6(f) + // ↑ + + // After i = 1 + // 2(u) - 3(u) - 4(f) - 5(f) - 6(f) + // ↑ + + // After i = 2 + // 2(u) - 3(u) - 4(u) - 5(f) - 6(f) + // ↑ + + // After i = 3 + // 2(u) - 3(u) - 4(u) - 5(u) - 6(f) + // ↑ + + // After i = 4 + // 2(u) - 3(u) - 4(u) - 5(u) - 6(u) + // ↑ + + // Check that there are no more IPs + ip, err := RequestIP(network, nil) + if err == nil { + t.Fatalf("There shouldn't be any IP addresses at this point, got %s\n", ip) + } + + // Release some IPs in non-sequential order + if err := ReleaseIP(network, &expectedIPs[3]); err != nil { + t.Fatal(err) + } + // 2(u) - 3(u) - 4(u) - 5(f) - 6(u) + // ↑ + + if err := ReleaseIP(network, &expectedIPs[2]); err != nil { + t.Fatal(err) + } + // 2(u) - 3(u) - 4(f) - 5(f) - 6(u) + // ↑ + + if err := ReleaseIP(network, &expectedIPs[4]); err != nil { + t.Fatal(err) + } + // 2(u) - 3(u) - 4(f) - 5(f) - 6(f) + // ↑ + + // Make sure that IPs are reused in sequential order, starting + // with the first released IP + newIPs := make([]*net.IP, 3) + for i := 0; i < 3; i++ { + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + newIPs[i] = ip + } + // Before loop begin + // 2(u) - 3(u) - 4(f) - 5(f) - 6(f) + // ↑ + + // After i = 0 + // 2(u) - 3(u) - 4(f) - 5(u) - 6(f) + // ↑ + + // After i = 1 + // 2(u) - 3(u) - 4(f) - 5(u) - 6(u) + // ↑ + + // After i = 2 + // 2(u) - 3(u) - 4(u) - 5(u) - 6(u) + // ↑ + + assertIPEquals(t, &expectedIPs[3], newIPs[0]) + assertIPEquals(t, &expectedIPs[4], newIPs[1]) + assertIPEquals(t, &expectedIPs[2], newIPs[2]) + + _, err = RequestIP(network, nil) + if err == nil { + t.Fatal("There shouldn't be any IP addresses at this point") + } +} + +func assertIPEquals(t *testing.T, ip1, ip2 *net.IP) { + if !ip1.Equal(*ip2) { + t.Fatalf("Expected IP %s, got %s", ip1, ip2) + } +} + +func AssertOverlap(CIDRx string, CIDRy string, t *testing.T) { + _, netX, _ := net.ParseCIDR(CIDRx) + _, netY, _ := net.ParseCIDR(CIDRy) + if !networkOverlaps(netX, netY) { + t.Errorf("%v and %v should overlap", netX, netY) + } +} + +func AssertNoOverlap(CIDRx string, CIDRy string, t *testing.T) { + _, netX, _ := net.ParseCIDR(CIDRx) + _, netY, _ := net.ParseCIDR(CIDRy) + if networkOverlaps(netX, netY) { + t.Errorf("%v and %v should not overlap", netX, netY) + } +} + +func TestNetworkOverlaps(t *testing.T) { + //netY starts at same IP and ends within netX + AssertOverlap("172.16.0.1/24", "172.16.0.1/25", t) + //netY starts within netX and ends at same IP + AssertOverlap("172.16.0.1/24", "172.16.0.128/25", t) + //netY starts and ends within netX + AssertOverlap("172.16.0.1/24", "172.16.0.64/25", t) + //netY starts at same IP and ends outside of netX + AssertOverlap("172.16.0.1/24", "172.16.0.1/23", t) + //netY starts before and ends at same IP of netX + AssertOverlap("172.16.1.1/24", "172.16.0.1/23", t) + //netY starts before and ends outside of netX + AssertOverlap("172.16.1.1/24", "172.16.0.1/23", t) + //netY starts and ends before netX + AssertNoOverlap("172.16.1.1/25", "172.16.0.1/24", t) + //netX starts and ends before netY + AssertNoOverlap("172.16.1.1/25", "172.16.2.1/24", t) +} + +func TestCheckRouteOverlaps(t *testing.T) { + routesData := []string{"10.0.2.0/32", "10.0.3.0/24", "10.0.42.0/24", "172.16.42.0/24", "192.168.142.0/24"} + + routes := []netlink.Route{} + for _, addr := range routesData { + _, netX, _ := net.ParseCIDR(addr) + routes = append(routes, netlink.Route{IPNet: netX}) + } + + _, netX, _ := net.ParseCIDR("172.16.0.1/24") + if err := checkRouteOverlaps(routes, netX); err != nil { + t.Fatal(err) + } + + _, netX, _ = net.ParseCIDR("10.0.2.0/24") + if err := checkRouteOverlaps(routes, netX); err == nil { + t.Fatalf("10.0.2.0/24 and 10.0.2.0 should overlap but it doesn't") + } +} + +func TestCheckNameserverOverlaps(t *testing.T) { + nameservers := []string{"10.0.2.3/32", "192.168.102.1/32"} + + _, netX, _ := net.ParseCIDR("10.0.2.3/32") + + if err := checkNameserverOverlaps(nameservers, netX); err == nil { + t.Fatalf("%s should overlap 10.0.2.3/32 but doesn't", netX) + } + + _, netX, _ = net.ParseCIDR("192.168.102.2/32") + + if err := checkNameserverOverlaps(nameservers, netX); err != nil { + t.Fatalf("%s should not overlap %v but it does", netX, nameservers) + } +} diff --git a/networkdriver/network.go b/networkdriver/network.go new file mode 100644 index 0000000000..e74734d55f --- /dev/null +++ b/networkdriver/network.go @@ -0,0 +1 @@ +package networkdriver From 9d11db0f8c2be614f18eec856f11edbff5da17fe Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 23 Jan 2014 02:22:13 -0800 Subject: [PATCH 11/12] Update code for new test cases Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- networkdriver/ipallocator/allocator.go | 35 +++++++++------------ networkdriver/ipallocator/allocator_test.go | 6 ++-- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/networkdriver/ipallocator/allocator.go b/networkdriver/ipallocator/allocator.go index e6d39446df..8b7c986461 100644 --- a/networkdriver/ipallocator/allocator.go +++ b/networkdriver/ipallocator/allocator.go @@ -19,7 +19,7 @@ var ( ErrNetworkAlreadyAllocated = errors.New("requested network overlaps with existing network") ErrNetworkAlreadyRegisterd = errors.New("requested network is already registered") ErrNetworkOverlapsWithNameservers = errors.New("requested network overlaps with nameserver") - ErrNoAvailableIps = errors.New("no available ips on network") + ErrNoAvailableIPs = errors.New("no available ip addresses on network") ErrIPAlreadyAllocated = errors.New("ip already allocated") lock = sync.Mutex{} @@ -100,11 +100,11 @@ func getNextIp(network *net.IPNet) (*net.IP, error) { ownIP = ipToInt(&network.IP) available = availableIPS[n] allocated = allocatedIPs[n] - - first, _ = networkRange(network) - base = ipToInt(&first) - - pos = int32(available.Pop()) + first, _ = networkRange(network) + base = ipToInt(&first) + size = int(networkSize(network.Mask)) + max = int32(size - 2) // size -1 for the broadcast address, -1 for the gateway address + pos = int32(available.Pop()) ) // We pop and push the position not the ip @@ -115,29 +115,22 @@ func getNextIp(network *net.IPNet) (*net.IP, error) { return ip, nil } - var ( - size = int(networkSize(network.Mask)) - max = int32(size - 2) // size -1 for the broadcast address, -1 for the gateway address - ) - - if pos = int32(allocated.PullBack()); pos == 0 { - pos = 1 - } - + pos = int32(allocated.PullBack()) for i := int32(0); i < max; i++ { - next := int32(base + pos) pos = pos%max + 1 + next := int32(base + pos) if next == ownIP { continue } - ip := intToIP(next) - allocated.Push(int(pos)) - - return ip, nil + if !allocated.Exists(int(pos)) { + ip := intToIP(next) + allocated.Push(int(pos)) + return ip, nil + } } - return nil, ErrNoAvailableIps + return nil, ErrNoAvailableIPs } func registerIP(network *net.IPNet, ip *net.IP) error { diff --git a/networkdriver/ipallocator/allocator_test.go b/networkdriver/ipallocator/allocator_test.go index d1a58a021d..2e2d463638 100644 --- a/networkdriver/ipallocator/allocator_test.go +++ b/networkdriver/ipallocator/allocator_test.go @@ -386,9 +386,9 @@ func TestIPAllocator(t *testing.T) { // 2(u) - 3(u) - 4(u) - 5(u) - 6(u) // ↑ - assertIPEquals(t, &expectedIPs[3], newIPs[0]) - assertIPEquals(t, &expectedIPs[4], newIPs[1]) - assertIPEquals(t, &expectedIPs[2], newIPs[2]) + assertIPEquals(t, &expectedIPs[2], newIPs[0]) + assertIPEquals(t, &expectedIPs[3], newIPs[1]) + assertIPEquals(t, &expectedIPs[4], newIPs[2]) _, err = RequestIP(network, nil) if err == nil { From ea5b3e193ba40b65c38c774e9abb85340c52e8c6 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 23 Jan 2014 05:22:32 -0800 Subject: [PATCH 12/12] Finalize core changes with new package Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- network.go | 13 +++- networkdriver/ipallocator/allocator.go | 68 ++++++++++++++++++--- networkdriver/ipallocator/allocator_test.go | 2 + 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/network.go b/network.go index 4987140402..7414b348a2 100644 --- a/network.go +++ b/network.go @@ -60,10 +60,13 @@ func CreateBridgeIface(config *DaemonConfig) error { var ifaceAddr string if len(config.BridgeIp) != 0 { - _, _, err := net.ParseCIDR(config.BridgeIp) + _, dockerNetwork, err := net.ParseCIDR(config.BridgeIp) if err != nil { return err } + if err := ipallocator.RegisterNetwork(dockerNetwork, nameservers); err != nil { + return err + } ifaceAddr = config.BridgeIp } else { for _, addr := range addrs { @@ -534,6 +537,7 @@ func newNetworkManager(config *DaemonConfig) (*NetworkManager, error) { return manager, nil } + var network *net.IPNet addr, err := getIfaceAddr(config.BridgeIface) if err != nil { // If the iface is not found, try to create it @@ -544,8 +548,13 @@ func newNetworkManager(config *DaemonConfig) (*NetworkManager, error) { if err != nil { return nil, err } + network = addr.(*net.IPNet) + } else { + network = addr.(*net.IPNet) + if err := ipallocator.RegisterExistingNetwork(network); err != nil { + return nil, err + } } - network := addr.(*net.IPNet) // Configure iptables for link support if config.EnableIptables { diff --git a/networkdriver/ipallocator/allocator.go b/networkdriver/ipallocator/allocator.go index 8b7c986461..cca8cdb05a 100644 --- a/networkdriver/ipallocator/allocator.go +++ b/networkdriver/ipallocator/allocator.go @@ -21,12 +21,16 @@ var ( ErrNetworkOverlapsWithNameservers = errors.New("requested network overlaps with nameserver") ErrNoAvailableIPs = errors.New("no available ip addresses on network") ErrIPAlreadyAllocated = errors.New("ip already allocated") + ErrNetworkNotRegistered = errors.New("network not registered") lock = sync.Mutex{} allocatedIPs = networkSet{} availableIPS = networkSet{} ) +// RegisterNetwork registers a new network with the allocator +// and validates that it contains a valid ip that does not overlap +// with existing routes and nameservers func RegisterNetwork(network *net.IPNet, nameservers []string) error { lock.Lock() defer lock.Unlock() @@ -47,19 +51,36 @@ func RegisterNetwork(network *net.IPNet, nameservers []string) error { if err := checkNameserverOverlaps(nameservers, network); err != nil { return err } + return RegisterExistingNetwork(network) +} +// RegisterExistingNetwork registers an exising network created +// for use with the allocator but does not perform any validation +func RegisterExistingNetwork(network *net.IPNet) error { n := newIPNet(network) - allocatedIPs[n] = &iPSet{} - availableIPS[n] = &iPSet{} + if _, exists := allocatedIPs[n]; !exists { + allocatedIPs[n] = &iPSet{} + } + if _, exists := availableIPS[n]; !exists { + availableIPS[n] = &iPSet{} + } return nil } +// RequestIP requests an available ip from the given network. It +// will return the next available ip if the ip provided is nil. If the +// ip provided is not nil it will validate that the provided ip is available +// for use or return an error func RequestIP(network *net.IPNet, ip *net.IP) (*net.IP, error) { lock.Lock() defer lock.Unlock() + if !networkExists(network) { + return nil, ErrNetworkNotRegistered + } + if ip == nil { next, err := getNextIp(network) if err != nil { @@ -74,18 +95,21 @@ func RequestIP(network *net.IPNet, ip *net.IP) (*net.IP, error) { return ip, nil } +// ReleaseIP adds the provided ip back into the pool of +// available ips to be returned for use. func ReleaseIP(network *net.IPNet, ip *net.IP) error { lock.Lock() defer lock.Unlock() + if !networkExists(network) { + return ErrNetworkNotRegistered + } + var ( - first, _ = networkRange(network) - base = ipToInt(&first) n = newIPNet(network) existing = allocatedIPs[n] available = availableIPS[n] - i = ipToInt(ip) - pos = i - base + pos = getPosition(network, ip) ) existing.Remove(int(pos)) @@ -94,6 +118,19 @@ func ReleaseIP(network *net.IPNet, ip *net.IP) error { return nil } +// convert the ip into the position in the subnet. Only +// position are saved in the set +func getPosition(network *net.IPNet, ip *net.IP) int32 { + var ( + first, _ = networkRange(network) + base = ipToInt(&first) + i = ipToInt(ip) + ) + return i - base +} + +// return an available ip if one is currently available. If not, +// return the next available ip for the nextwork func getNextIp(network *net.IPNet) (*net.IP, error) { var ( n = newIPNet(network) @@ -134,11 +171,18 @@ func getNextIp(network *net.IPNet) (*net.IP, error) { } func registerIP(network *net.IPNet, ip *net.IP) error { - existing := allocatedIPs[newIPNet(network)] - // checking position not ip - if existing.Exists(int(ipToInt(ip))) { + var ( + n = newIPNet(network) + existing = allocatedIPs[n] + available = availableIPS[n] + pos = getPosition(network, ip) + ) + + if existing.Exists(int(pos)) { return ErrIPAlreadyAllocated } + available.Remove(int(pos)) + return nil } @@ -241,3 +285,9 @@ func checkNameserverOverlaps(nameservers []string, toCheck *net.IPNet) error { } return nil } + +func networkExists(network *net.IPNet) bool { + n := newIPNet(network) + _, exists := allocatedIPs[n] + return exists +} diff --git a/networkdriver/ipallocator/allocator_test.go b/networkdriver/ipallocator/allocator_test.go index 2e2d463638..f574dfda70 100644 --- a/networkdriver/ipallocator/allocator_test.go +++ b/networkdriver/ipallocator/allocator_test.go @@ -386,6 +386,8 @@ func TestIPAllocator(t *testing.T) { // 2(u) - 3(u) - 4(u) - 5(u) - 6(u) // ↑ + // Reordered these because the new set will always return the + // lowest ips first and not in the order that they were released assertIPEquals(t, &expectedIPs[2], newIPs[0]) assertIPEquals(t, &expectedIPs[3], newIPs[1]) assertIPEquals(t, &expectedIPs[4], newIPs[2])