From c561212b837735213f146c2fac291f84dbcbc8c3 Mon Sep 17 00:00:00 2001 From: Paul Nasrat Date: Sat, 21 Dec 2013 11:02:06 -0500 Subject: [PATCH] Extract cgroups utilities to own submodule. Use mountinfo rather than for cgroups parsing. Make helper method private and change name. Makes method naming more explicit rather than GetThisCgroup. Use upstream term subsystem rather than cgroupType. --- cgroups/MAINTAINERS | 1 + cgroups/cgroups.go | 101 ++++++++++++++++++++++++++++++++++++++++ cgroups/cgroups_test.go | 27 +++++++++++ mount/mount.go | 6 ++- mount/mountinfo.go | 49 +++++++++++++------ runtime.go | 5 +- server.go | 3 +- utils/utils.go | 78 ------------------------------- 8 files changed, 175 insertions(+), 95 deletions(-) create mode 100644 cgroups/MAINTAINERS create mode 100644 cgroups/cgroups.go create mode 100644 cgroups/cgroups_test.go diff --git a/cgroups/MAINTAINERS b/cgroups/MAINTAINERS new file mode 100644 index 0000000000..1e998f8ac1 --- /dev/null +++ b/cgroups/MAINTAINERS @@ -0,0 +1 @@ +Michael Crosby (@crosbymichael) diff --git a/cgroups/cgroups.go b/cgroups/cgroups.go new file mode 100644 index 0000000000..30de8d4d1e --- /dev/null +++ b/cgroups/cgroups.go @@ -0,0 +1,101 @@ +package cgroups + +import ( + "bufio" + "fmt" + "github.com/dotcloud/docker/mount" + "io" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" +) + +// https://www.kernel.org/doc/Documentation/cgroups/cgroups.txt + +func FindCgroupMountpoint(subsystem string) (string, error) { + mounts, err := mount.GetMounts() + if err != nil { + return "", err + } + + for _, mount := range mounts { + if mount.Fstype == "cgroup" { + for _, opt := range strings.Split(mount.VfsOpts, ",") { + if opt == subsystem { + return mount.Mountpoint, nil + } + } + } + } + + return "", fmt.Errorf("cgroup mountpoint not found for %s", subsystem) +} + +// Returns the relative path to the cgroup docker is running in. +func getThisCgroupDir(subsystem string) (string, error) { + f, err := os.Open("/proc/self/cgroup") + if err != nil { + return "", err + } + defer f.Close() + + return parseCgroupFile(subsystem, f) +} + +func parseCgroupFile(subsystem string, r io.Reader) (string, error) { + s := bufio.NewScanner(r) + + for s.Scan() { + if err := s.Err(); err != nil { + return "", err + } + text := s.Text() + parts := strings.Split(text, ":") + if parts[1] == subsystem { + return parts[2], nil + } + } + return "", fmt.Errorf("cgroup '%s' not found in /proc/self/cgroup", subsystem) +} + +// Returns a list of pids for the given container. +func GetPidsForContainer(id string) ([]int, error) { + pids := []int{} + + // memory is chosen randomly, any cgroup used by docker works + subsystem := "memory" + + cgroupRoot, err := FindCgroupMountpoint(subsystem) + if err != nil { + return pids, err + } + + cgroupDir, err := getThisCgroupDir(subsystem) + if err != nil { + return pids, err + } + + filename := filepath.Join(cgroupRoot, cgroupDir, id, "tasks") + if _, err := os.Stat(filename); os.IsNotExist(err) { + // With more recent lxc versions use, cgroup will be in lxc/ + filename = filepath.Join(cgroupRoot, cgroupDir, "lxc", id, "tasks") + } + + output, err := ioutil.ReadFile(filename) + if err != nil { + return pids, err + } + for _, p := range strings.Split(string(output), "\n") { + if len(p) == 0 { + continue + } + pid, err := strconv.Atoi(p) + if err != nil { + return pids, fmt.Errorf("Invalid pid '%s': %s", p, err) + } + pids = append(pids, pid) + } + return pids, nil +} diff --git a/cgroups/cgroups_test.go b/cgroups/cgroups_test.go new file mode 100644 index 0000000000..537336a493 --- /dev/null +++ b/cgroups/cgroups_test.go @@ -0,0 +1,27 @@ +package cgroups + +import ( + "bytes" + "testing" +) + +const ( + cgroupsContents = `11:hugetlb:/ +10:perf_event:/ +9:blkio:/ +8:net_cls:/ +7:freezer:/ +6:devices:/ +5:memory:/ +4:cpuacct,cpu:/ +3:cpuset:/ +2:name=systemd:/user.slice/user-1000.slice/session-16.scope` +) + +func TestParseCgroups(t *testing.T) { + r := bytes.NewBuffer([]byte(cgroupsContents)) + _, err := parseCgroupFile("blkio", r) + if err != nil { + t.Fatal(err) + } +} diff --git a/mount/mount.go b/mount/mount.go index 253a4681ef..b087293a9d 100644 --- a/mount/mount.go +++ b/mount/mount.go @@ -4,6 +4,10 @@ import ( "time" ) +func GetMounts() ([]*MountInfo, error) { + return parseMountTable() +} + // Looks at /proc/self/mountinfo to determine of the specified // mountpoint has been mounted func Mounted(mountpoint string) (bool, error) { @@ -14,7 +18,7 @@ func Mounted(mountpoint string) (bool, error) { // Search the table for the mountpoint for _, e := range entries { - if e.mountpoint == mountpoint { + if e.Mountpoint == mountpoint { return true, nil } } diff --git a/mount/mountinfo.go b/mount/mountinfo.go index f4fd24b633..32996f05c8 100644 --- a/mount/mountinfo.go +++ b/mount/mountinfo.go @@ -5,22 +5,35 @@ import ( "fmt" "io" "os" + "strings" ) const ( - // We only parse upto the mountinfo because that is all we - // care about right now - mountinfoFormat = "%d %d %d:%d %s %s %s" + /* 36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue + (1)(2)(3) (4) (5) (6) (7) (8) (9) (10) (11) + + (1) mount ID: unique identifier of the mount (may be reused after umount) + (2) parent ID: ID of parent (or of self for the top of the mount tree) + (3) major:minor: value of st_dev for files on filesystem + (4) root: root of the mount within the filesystem + (5) mount point: mount point relative to the process's root + (6) mount options: per mount options + (7) optional fields: zero or more fields of the form "tag[:value]" + (8) separator: marks the end of the optional fields + (9) filesystem type: name of filesystem of the form "type[.subtype]" + (10) mount source: filesystem specific information or "none" + (11) super options: per super block options*/ + mountinfoFormat = "%d %d %d:%d %s %s %s " ) -// Represents one line from /proc/self/mountinfo -type procEntry struct { - id, parent, major, minor int - source, mountpoint, opts string +type MountInfo struct { + Id, Parent, Major, Minor int + Root, Mountpoint, Opts string + Fstype, Source, VfsOpts string } // Parse /proc/self/mountinfo because comparing Dev and ino does not work from bind mounts -func parseMountTable() ([]*procEntry, error) { +func parseMountTable() ([]*MountInfo, error) { f, err := os.Open("/proc/self/mountinfo") if err != nil { return nil, err @@ -30,10 +43,10 @@ func parseMountTable() ([]*procEntry, error) { return parseInfoFile(f) } -func parseInfoFile(r io.Reader) ([]*procEntry, error) { +func parseInfoFile(r io.Reader) ([]*MountInfo, error) { var ( s = bufio.NewScanner(r) - out = []*procEntry{} + out = []*MountInfo{} ) for s.Scan() { @@ -42,14 +55,24 @@ func parseInfoFile(r io.Reader) ([]*procEntry, error) { } var ( - p = &procEntry{} + p = &MountInfo{} text = s.Text() ) + if _, err := fmt.Sscanf(text, mountinfoFormat, - &p.id, &p.parent, &p.major, &p.minor, - &p.source, &p.mountpoint, &p.opts); err != nil { + &p.Id, &p.Parent, &p.Major, &p.Minor, + &p.Root, &p.Mountpoint, &p.Opts); err != nil { return nil, fmt.Errorf("Scanning '%s' failed: %s", text, err) } + // Safe as mountinfo encodes mountpoints with spaces as \040. + index := strings.Index(text, " - ") + postSeparatorFields := strings.Fields(text[index+3:]) + if len(postSeparatorFields) != 3 { + return nil, fmt.Errorf("Error did not find 3 fields post '-' in '%s'", text) + } + p.Fstype = postSeparatorFields[0] + p.Source = postSeparatorFields[1] + p.VfsOpts = postSeparatorFields[2] out = append(out, p) } return out, nil diff --git a/runtime.go b/runtime.go index 7c7f722c5d..b1d8988607 100644 --- a/runtime.go +++ b/runtime.go @@ -4,11 +4,12 @@ import ( "container/list" "fmt" "github.com/dotcloud/docker/archive" - "github.com/dotcloud/docker/pkg/graphdb" + "github.com/dotcloud/docker/cgroups" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/graphdriver/aufs" _ "github.com/dotcloud/docker/graphdriver/devmapper" _ "github.com/dotcloud/docker/graphdriver/vfs" + "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -332,7 +333,7 @@ func (runtime *Runtime) restore() error { // FIXME: comment please! func (runtime *Runtime) UpdateCapabilities(quiet bool) { - if cgroupMemoryMountpoint, err := utils.FindCgroupMountpoint("memory"); err != nil { + if cgroupMemoryMountpoint, err := cgroups.FindCgroupMountpoint("memory"); err != nil { if !quiet { log.Printf("WARNING: %s\n", err) } diff --git a/server.go b/server.go index 9f6842dbae..6445c540b0 100644 --- a/server.go +++ b/server.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/cgroups" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/registry" @@ -701,7 +702,7 @@ func (srv *Server) ContainerTop(name, psArgs string) (*APITop, error) { if !container.State.IsRunning() { return nil, fmt.Errorf("Container %s is not running", name) } - pids, err := utils.GetPidsForContainer(container.ID) + pids, err := cgroups.GetPidsForContainer(container.ID) if err != nil { return nil, err } diff --git a/utils/utils.go b/utils/utils.go index ffe90ac734..b0ba43d377 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -583,28 +583,6 @@ func CompareKernelVersion(a, b *KernelVersionInfo) int { return 0 } -func FindCgroupMountpoint(cgroupType string) (string, error) { - output, err := ioutil.ReadFile("/proc/mounts") - if err != nil { - return "", err - } - - // /proc/mounts has 6 fields per line, one mount per line, e.g. - // cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0 - for _, line := range strings.Split(string(output), "\n") { - parts := strings.Split(line, " ") - if len(parts) == 6 && parts[2] == "cgroup" { - for _, opt := range strings.Split(parts[3], ",") { - if opt == cgroupType { - return parts[1], nil - } - } - } - } - - return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) -} - func GetKernelVersion() (*KernelVersionInfo, error) { var ( err error @@ -1157,59 +1135,3 @@ func CopyFile(src, dst string) (int64, error) { defer df.Close() return io.Copy(df, sf) } - -// Returns the relative path to the cgroup docker is running in. -func GetThisCgroup(cgroupType string) (string, error) { - output, err := ioutil.ReadFile("/proc/self/cgroup") - if err != nil { - return "", err - } - for _, line := range strings.Split(string(output), "\n") { - parts := strings.Split(line, ":") - // any type used by docker should work - if parts[1] == cgroupType { - return parts[2], nil - } - } - return "", fmt.Errorf("cgroup '%s' not found in /proc/self/cgroup", cgroupType) -} - -// Returns a list of pids for the given container. -func GetPidsForContainer(id string) ([]int, error) { - pids := []int{} - - // memory is chosen randomly, any cgroup used by docker works - cgroupType := "memory" - - cgroupRoot, err := FindCgroupMountpoint(cgroupType) - if err != nil { - return pids, err - } - - cgroupThis, err := GetThisCgroup(cgroupType) - if err != nil { - return pids, err - } - - filename := filepath.Join(cgroupRoot, cgroupThis, id, "tasks") - if _, err := os.Stat(filename); os.IsNotExist(err) { - // With more recent lxc versions use, cgroup will be in lxc/ - filename = filepath.Join(cgroupRoot, cgroupThis, "lxc", id, "tasks") - } - - output, err := ioutil.ReadFile(filename) - if err != nil { - return pids, err - } - for _, p := range strings.Split(string(output), "\n") { - if len(p) == 0 { - continue - } - pid, err := strconv.Atoi(p) - if err != nil { - return pids, fmt.Errorf("Invalid pid '%s': %s", p, err) - } - pids = append(pids, pid) - } - return pids, nil -}