From a02b7f162016dbd186d21f0549a3292b8eac950b Mon Sep 17 00:00:00 2001 From: Abin Shahab Date: Tue, 14 Oct 2014 02:01:30 +0000 Subject: [PATCH] Added network stats to raw handler Raw handler now parses an optional json file with the network interface information, and emits network stats. --- cadvisor.go | 6 - container/lxc/factory.go | 103 ----- container/lxc/handler.go | 400 ------------------ container/{lxc => raw}/container_desc.go | 19 +- container/{lxc => raw}/container_desc_test.go | 4 +- container/raw/handler.go | 28 +- .../{lxc => raw}/test_resources/cdesc.json | 0 7 files changed, 38 insertions(+), 522 deletions(-) delete mode 100644 container/lxc/factory.go delete mode 100644 container/lxc/handler.go rename container/{lxc => raw}/container_desc.go (75%) rename container/{lxc => raw}/container_desc_test.go (90%) rename container/{lxc => raw}/test_resources/cdesc.json (100%) diff --git a/cadvisor.go b/cadvisor.go index ac81ab04..bf499ea8 100644 --- a/cadvisor.go +++ b/cadvisor.go @@ -27,7 +27,6 @@ import ( "github.com/golang/glog" "github.com/google/cadvisor/api" "github.com/google/cadvisor/container/docker" - "github.com/google/cadvisor/container/lxc" "github.com/google/cadvisor/container/raw" "github.com/google/cadvisor/healthz" "github.com/google/cadvisor/info" @@ -62,11 +61,6 @@ func main() { glog.Errorf("Docker registration failed: %v.", err) } - // Register the lxc driver. - if err := lxc.Register(containerManager); err != nil { - glog.Fatalf("lxc registration failed: %v.", err) - } - // Register the raw driver. if err := raw.Register(containerManager); err != nil { glog.Fatalf("raw registration failed: %v.", err) diff --git a/container/lxc/factory.go b/container/lxc/factory.go deleted file mode 100644 index d9d52fa3..00000000 --- a/container/lxc/factory.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2014 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Creates handler to handle containers launched by lxc (under /var/lib/docker/lxc) -package lxc - -import ( - "fmt" - "strings" - "github.com/docker/libcontainer/cgroups" - "github.com/golang/glog" - "github.com/google/cadvisor/container" - "github.com/google/cadvisor/info" -) - -type cgroupSubsystems struct { - // Cgroup subsystem mounts. - mounts []cgroups.Mount - - // Cgroup subsystem to their mount location. - mountPoints map[string]string -} - -type lxcFactory struct { - // Factory for machine information. - machineInfoFactory info.MachineInfoFactory - cgroupSubsystems *cgroupSubsystems -} - -func (self *lxcFactory) String() string { - return "lxc" -} - -func (self *lxcFactory) NewContainerHandler(name string) (container.ContainerHandler, error) { - fmt.Printf("Invoking NewLxcContainerhandler for %s by %q\n", name, self.String()) - return newLxcContainerHandler(name, self.cgroupSubsystems, self.machineInfoFactory) -} - -// The lxc factory can handle any container. -func (self *lxcFactory) CanHandle(name string) (bool, error) { - if name == "/lxc" { - return true, nil - } else if strings.HasPrefix(name, "/lxc/") { - return true, nil - } - return false, nil -} - -func Register(machineInfoFactory info.MachineInfoFactory) error { - // Get all cgroup mounts. - allCgroups, err := cgroups.GetCgroupMounts() - if err != nil { - return err - } - if len(allCgroups) == 0 { - return fmt.Errorf("failed to find cgroup mounts for the lxc factory") - } - - // Trim the mounts to only the subsystems we care about. - supportedCgroups := make([]cgroups.Mount, 0, len(allCgroups)) - mountPoints := make(map[string]string, len(allCgroups)) - for _, mount := range allCgroups { - for _, subsystem := range mount.Subsystems { - if _, ok := supportedSubsystems[subsystem]; ok { - supportedCgroups = append(supportedCgroups, mount) - mountPoints[subsystem] = mount.Mountpoint - } - } - } - if len(supportedCgroups) == 0 { - return fmt.Errorf("failed to find supported cgroup mounts for the lxc factory") - } - - glog.Infof("Registering Lxc factory") - factory := &lxcFactory{ - machineInfoFactory: machineInfoFactory, - cgroupSubsystems: &cgroupSubsystems{ - mounts: supportedCgroups, - mountPoints: mountPoints, - }, - } - container.RegisterContainerHandlerFactory(factory) - return nil -} - -// Cgroup subsystems we support listing (should be the minimal set we need stats from). -var supportedSubsystems map[string]struct{} = map[string]struct{}{ - "cpu": {}, - "cpuacct": {}, - "memory": {}, - "cpuset": {}, -} diff --git a/container/lxc/handler.go b/container/lxc/handler.go deleted file mode 100644 index a9d16bdb..00000000 --- a/container/lxc/handler.go +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright 2014 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Handles containers launched by lxc (under /var/lib/docker/lxc) -// Gathers stats about memory, cpu, network, and diskio -package lxc - -import ( - "code.google.com/p/go.exp/inotify" - "flag" - "fmt" - "io/ioutil" - "path" - "strconv" - "strings" - dockerlibcontainer "github.com/docker/libcontainer" - "github.com/docker/libcontainer/cgroups" - cgroup_fs "github.com/docker/libcontainer/cgroups/fs" - "github.com/docker/libcontainer/network" - "github.com/golang/glog" - "github.com/google/cadvisor/container" - "github.com/google/cadvisor/container/libcontainer" - "github.com/google/cadvisor/fs" - "github.com/google/cadvisor/info" - "github.com/google/cadvisor/utils" -) - -var containersDesc = flag.String("cDesc", "/etc/docker/cdesc.json", "container description file") - -type lxcContainerHandler struct { - name string - cgroup *cgroups.Cgroup - cgroupSubsystems *cgroupSubsystems - machineInfoFactory info.MachineInfoFactory - watcher *inotify.Watcher - stopWatcher chan error - watches map[string]struct{} - fsInfo fs.FsInfo - network_interface *NetworkInterface -} - -func newLxcContainerHandler(name string, cgroupSubsystems *cgroupSubsystems, machineInfoFactory info.MachineInfoFactory) (container.ContainerHandler, error) { - fsInfo, err := fs.NewFsInfo() - if err != nil { - return nil, err - } - cDesc, err := Unmarshal(*containersDesc) - var network_interface *NetworkInterface - for _, container := range cDesc.All_hosts { - cName := "/lxc/" + container.Id - glog.Infof("container %s Name %s \n\n", container, name) - if cName == name { - network_interface = container.Network_interface - fmt.Printf("Found network interface %s \n\n", network_interface) - break - } - } - return &lxcContainerHandler{ - name: name, - cgroup: &cgroups.Cgroup{ - Parent: "/", - Name: name, - }, - cgroupSubsystems: cgroupSubsystems, - machineInfoFactory: machineInfoFactory, - stopWatcher: make(chan error), - watches: make(map[string]struct{}), - fsInfo: fsInfo, - network_interface: network_interface, - }, nil -} - -func (self *lxcContainerHandler) ContainerReference() (info.ContainerReference, error) { - // We only know the container by its one name. - return info.ContainerReference{ - Name: self.name, - }, nil -} - -func readString(dirpath string, file string) string { - cgroupFile := path.Join(dirpath, file) - - // Ignore non-existent files - if !utils.FileExists(cgroupFile) { - return "" - } - - // Read - out, err := ioutil.ReadFile(cgroupFile) - if err != nil { - glog.Errorf("lxc driver: Failed to read %q: %s", cgroupFile, err) - return "" - } - return string(out) -} - -func readInt64(dirpath string, file string) uint64 { - out := readString(dirpath, file) - if out == "" { - return 0 - } - - val, err := strconv.ParseUint(strings.TrimSpace(out), 10, 64) - if err != nil { - glog.Errorf("lxc driver: Failed to parse int %q from file %q: %s", out, path.Join(dirpath, file), err) - return 0 - } - - return val -} - -func (self *lxcContainerHandler) GetSpec() (info.ContainerSpec, error) { - var spec info.ContainerSpec - - // The lxc driver assumes unified hierarchy containers. - - // Get machine info. - mi, err := self.machineInfoFactory.GetMachineInfo() - if err != nil { - return spec, err - } - - // CPU. - cpuRoot, ok := self.cgroupSubsystems.mountPoints["cpu"] - if ok { - cpuRoot = path.Join(cpuRoot, self.name) - if utils.FileExists(cpuRoot) { - spec.HasCpu = true - spec.Cpu.Limit = readInt64(cpuRoot, "cpu.shares") - } - } - - // Cpu Mask. - // This will fail for non-unified hierarchies. We'll return the whole machine mask in that case. - cpusetRoot, ok := self.cgroupSubsystems.mountPoints["cpuset"] - if ok { - cpusetRoot = path.Join(cpusetRoot, self.name) - if utils.FileExists(cpusetRoot) { - spec.HasCpu = true - spec.Cpu.Mask = readString(cpusetRoot, "cpuset.cpus") - if spec.Cpu.Mask == "" { - spec.Cpu.Mask = fmt.Sprintf("0-%d", mi.NumCores-1) - } - } - } - - // Memory. - memoryRoot, ok := self.cgroupSubsystems.mountPoints["memory"] - if ok { - memoryRoot = path.Join(memoryRoot, self.name) - if utils.FileExists(memoryRoot) { - spec.HasMemory = true - spec.Memory.Limit = readInt64(memoryRoot, "memory.limit_in_bytes") - spec.Memory.SwapLimit = readInt64(memoryRoot, "memory.memsw.limit_in_bytes") - } - } - - // Fs. - if self.name == "/" { - spec.HasFilesystem = true - } - return spec, nil -} - -func (self *lxcContainerHandler) GetStats() (*info.ContainerStats, error) { - var stats *info.ContainerStats - var err error - if self.network_interface != nil { - n := network.NetworkState{VethHost: self.network_interface.VethHost, VethChild: self.network_interface.VethChild, NsPath: "unknown"} - s := dockerlibcontainer.State{NetworkState: n} - stats, err = libcontainer.GetStats(self.cgroup, &s) - } else { - stats, err = libcontainer.GetStatsCgroupOnly(self.cgroup) - } - if err != nil { - return nil, err - } - // Get Filesystem information only for the root cgroup. - if self.name == "/" { - stats.Filesystem, err = self.fsInfo.GetFsStats() - if err != nil { - return nil, err - } - } - - return stats, nil -} - -// Lists all directories under "path" and outputs the results as children of "parent". -func listDirectories(dirpath string, parent string, recursive bool, output map[string]struct{}) error { - // Ignore if this hierarchy does not exist. - if !utils.FileExists(dirpath) { - return nil - } - - entries, err := ioutil.ReadDir(dirpath) - if err != nil { - return err - } - for _, entry := range entries { - // We only grab directories. - if entry.IsDir() { - name := path.Join(parent, entry.Name()) - output[name] = struct{}{} - - // List subcontainers if asked to. - if recursive { - err := listDirectories(path.Join(dirpath, entry.Name()), name, true, output) - if err != nil { - return err - } - } - } - } - return nil -} - -func (self *lxcContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) { - containers := make(map[string]struct{}) - for _, subsystem := range self.cgroupSubsystems.mounts { - err := listDirectories(path.Join(subsystem.Mountpoint, self.name), self.name, listType == container.ListRecursive, containers) - if err != nil { - return nil, err - } - } - - // Make into container references. - ret := make([]info.ContainerReference, 0, len(containers)) - for cont := range containers { - ret = append(ret, info.ContainerReference{ - Name: cont, - }) - } - - return ret, nil -} - -func (self *lxcContainerHandler) ListThreads(listType container.ListType) ([]int, error) { - // TODO(vmarmol): Implement - return nil, nil -} - -func (self *lxcContainerHandler) ListProcesses(listType container.ListType) ([]int, error) { - return cgroup_fs.GetPids(self.cgroup) -} - -func (self *lxcContainerHandler) watchDirectory(dir string, containerName string) error { - err := self.watcher.AddWatch(dir, inotify.IN_CREATE|inotify.IN_DELETE|inotify.IN_MOVE) - if err != nil { - return err - } - self.watches[containerName] = struct{}{} - - // Watch subdirectories as well. - entries, err := ioutil.ReadDir(dir) - if err != nil { - return err - } - for _, entry := range entries { - if entry.IsDir() { - err = self.watchDirectory(path.Join(dir, entry.Name()), path.Join(containerName, entry.Name())) - if err != nil { - return err - } - } - } - return nil -} - -func (self *lxcContainerHandler) processEvent(event *inotify.Event, events chan container.SubcontainerEvent) error { - // Convert the inotify event type to a container create or delete. - var eventType container.SubcontainerEventType - switch { - case (event.Mask & inotify.IN_CREATE) > 0: - eventType = container.SubcontainerAdd - case (event.Mask & inotify.IN_DELETE) > 0: - eventType = container.SubcontainerDelete - case (event.Mask & inotify.IN_MOVED_FROM) > 0: - eventType = container.SubcontainerDelete - case (event.Mask & inotify.IN_MOVED_TO) > 0: - eventType = container.SubcontainerAdd - default: - // Ignore other events. - return nil - } - - // Derive the container name from the path name. - var containerName string - for _, mount := range self.cgroupSubsystems.mounts { - mountLocation := path.Clean(mount.Mountpoint) + "/" - if strings.HasPrefix(event.Name, mountLocation) { - containerName = event.Name[len(mountLocation)-1:] - break - } - } - if containerName == "" { - return fmt.Errorf("unable to detect container from watch event on directory %q", event.Name) - } - - // Maintain the watch for the new or deleted container. - switch { - case eventType == container.SubcontainerAdd: - // If we've already seen this event, return. - if _, ok := self.watches[containerName]; ok { - return nil - } - - // New container was created, watch it. - err := self.watchDirectory(event.Name, containerName) - if err != nil { - return err - } - case eventType == container.SubcontainerDelete: - // If we've already seen this event, return. - if _, ok := self.watches[containerName]; !ok { - return nil - } - delete(self.watches, containerName) - - // Container was deleted, stop watching for it. - err := self.watcher.RemoveWatch(event.Name) - if err != nil { - return err - } - default: - return fmt.Errorf("unknown event type %v", eventType) - } - - // Deliver the event. - events <- container.SubcontainerEvent{ - EventType: eventType, - Name: containerName, - } - - return nil -} - -func (self *lxcContainerHandler) WatchSubcontainers(events chan container.SubcontainerEvent) error { - // Lazily initialize the watcher so we don't use it when not asked to. - if self.watcher == nil { - w, err := inotify.NewWatcher() - if err != nil { - return err - } - self.watcher = w - } - - // Watch this container (all its cgroups) and all subdirectories. - for _, mnt := range self.cgroupSubsystems.mounts { - err := self.watchDirectory(path.Join(mnt.Mountpoint, self.name), self.name) - if err != nil { - return err - } - } - - // Process the events received from the kernel. - go func() { - for { - select { - case event := <-self.watcher.Event: - err := self.processEvent(event, events) - if err != nil { - glog.Warningf("Error while processing event (%+v): %v", event, err) - } - case err := <-self.watcher.Error: - glog.Warningf("Error while watching %q:", self.name, err) - case <-self.stopWatcher: - err := self.watcher.Close() - if err == nil { - self.stopWatcher <- err - self.watcher = nil - return - } - } - } - }() - - return nil -} - -func (self *lxcContainerHandler) StopWatchingSubcontainers() error { - if self.watcher == nil { - return fmt.Errorf("can't stop watch that has not started for container %q", self.name) - } - - // Rendezvous with the watcher thread. - self.stopWatcher <- nil - return <-self.stopWatcher -} diff --git a/container/lxc/container_desc.go b/container/raw/container_desc.go similarity index 75% rename from container/lxc/container_desc.go rename to container/raw/container_desc.go index c1f2f0f8..09b7ed23 100644 --- a/container/lxc/container_desc.go +++ b/container/raw/container_desc.go @@ -16,31 +16,32 @@ // an array of ContainerDesc structs, each with a container's id and network_interface // This allows collecting stats about network interfaces configured outside docker // and lxc -package lxc +package raw import ( + "flag" "encoding/json" "io/ioutil" ) - -type ContainersDesc struct { - All_hosts []ContainerDesc +var argContainersDesc = flag.String("cDescription", "/etc/docker/cdesc.json", "container description file") +type containersDesc struct { + All_hosts []containerDesc } -type ContainerDesc struct { +type containerDesc struct { Id string - Network_interface *NetworkInterface + Network_interface *networkInterface } -type NetworkInterface struct { +type networkInterface struct { VethHost string VethChild string NsPath string } -func Unmarshal(containerDescFile string) (ContainersDesc, error) { +func Unmarshal(containerDescFile string) (containersDesc, error) { dat, err := ioutil.ReadFile(containerDescFile) - var cDesc ContainersDesc + var cDesc containersDesc if err == nil { err = json.Unmarshal(dat, &cDesc) } diff --git a/container/lxc/container_desc_test.go b/container/raw/container_desc_test.go similarity index 90% rename from container/lxc/container_desc_test.go rename to container/raw/container_desc_test.go index f5cbd8cc..faecd88b 100644 --- a/container/lxc/container_desc_test.go +++ b/container/raw/container_desc_test.go @@ -1,4 +1,4 @@ -package lxc +package raw import ( "testing" @@ -6,11 +6,11 @@ import ( func TestUnmarshal(t *testing.T) { cDesc, err := Unmarshal("test_resources/cdesc.json") + if err != nil { t.Fatalf("Error in unmarshalling: %s", err) } - t.Logf("Cdesc: %s", cDesc) if cDesc.All_hosts[0].Network_interface.VethHost != "veth24031eth1" && cDesc.All_hosts[0].Network_interface.VethChild != "eth1" { t.Errorf("Cannot find network interface in %s", cDesc) diff --git a/container/raw/handler.go b/container/raw/handler.go index 9a3c521c..1233e17d 100644 --- a/container/raw/handler.go +++ b/container/raw/handler.go @@ -23,8 +23,10 @@ import ( "strings" "code.google.com/p/go.exp/inotify" + dockerlibcontainer "github.com/docker/libcontainer" "github.com/docker/libcontainer/cgroups" cgroup_fs "github.com/docker/libcontainer/cgroups/fs" + "github.com/docker/libcontainer/network" "github.com/golang/glog" "github.com/google/cadvisor/container" "github.com/google/cadvisor/container/libcontainer" @@ -42,6 +44,7 @@ type rawContainerHandler struct { stopWatcher chan error watches map[string]struct{} fsInfo fs.FsInfo + network_interface *networkInterface } func newRawContainerHandler(name string, cgroupSubsystems *cgroupSubsystems, machineInfoFactory info.MachineInfoFactory) (container.ContainerHandler, error) { @@ -49,6 +52,18 @@ func newRawContainerHandler(name string, cgroupSubsystems *cgroupSubsystems, mac if err != nil { return nil, err } + cDesc, err := Unmarshal(*argContainersDesc) + var network_interface *networkInterface + for _, container := range cDesc.All_hosts { + names := strings.SplitAfter(name, "/") + cName := names[len(names) - 1] + glog.Infof("container %s Name %s \n\n", container, name) + if cName == container.Id { + network_interface = container.Network_interface + fmt.Printf("Found network interface %s \n\n", network_interface) + break + } + } return &rawContainerHandler{ name: name, cgroup: &cgroups.Cgroup{ @@ -60,6 +75,7 @@ func newRawContainerHandler(name string, cgroupSubsystems *cgroupSubsystems, mac stopWatcher: make(chan error), watches: make(map[string]struct{}), fsInfo: fsInfo, + network_interface: network_interface, }, nil } @@ -156,9 +172,17 @@ func (self *rawContainerHandler) GetSpec() (info.ContainerSpec, error) { } func (self *rawContainerHandler) GetStats() (*info.ContainerStats, error) { - stats, err := libcontainer.GetStatsCgroupOnly(self.cgroup) + var stats *info.ContainerStats + var err error + if self.network_interface != nil { + n := network.NetworkState{VethHost: self.network_interface.VethHost, VethChild: self.network_interface.VethChild, NsPath: "unknown"} + s := dockerlibcontainer.State{NetworkState: n} + stats, err = libcontainer.GetStats(self.cgroup, &s) + } else { + stats, err = libcontainer.GetStatsCgroupOnly(self.cgroup) + } if err != nil { - return nil, err + return nil, err } // Get Filesystem information only for the root cgroup. if self.name == "/" { diff --git a/container/lxc/test_resources/cdesc.json b/container/raw/test_resources/cdesc.json similarity index 100% rename from container/lxc/test_resources/cdesc.json rename to container/raw/test_resources/cdesc.json