diff --git a/cadvisor.go b/cadvisor.go index ef07993c..33de78e4 100644 --- a/cadvisor.go +++ b/cadvisor.go @@ -78,6 +78,7 @@ var ( // List of metrics that can be ignored. ignoreWhitelist = container.MetricSet{ container.DiskUsageMetrics: struct{}{}, + container.DiskIOMetrics: struct{}{}, container.NetworkUsageMetrics: struct{}{}, container.NetworkTcpUsageMetrics: struct{}{}, container.NetworkUdpUsageMetrics: struct{}{}, @@ -115,7 +116,7 @@ func (ml *metricSetValue) Set(value string) error { } func init() { - flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'disk', 'network', 'tcp', 'udp', 'percpu', 'sched', 'process'. Note: tcp and udp are disabled by default due to high CPU usage.") + flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'disk', 'diskIO', network', 'tcp', 'udp', 'percpu', 'sched', 'process'.") // Default logging verbosity to V(2) flag.Set("v", "2") diff --git a/container/containerd/factory.go b/container/containerd/factory.go index 2e2bb333..2a7f9dd5 100644 --- a/container/containerd/factory.go +++ b/container/containerd/factory.go @@ -128,7 +128,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics return fmt.Errorf("failed to fetch containerd client version: %v", err) } - cgroupSubsystems, err := libcontainer.GetCgroupSubsystems() + cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics) if err != nil { return fmt.Errorf("failed to get cgroup subsystems: %v", err) } diff --git a/container/containerd/handler.go b/container/containerd/handler.go index 82aa8082..9e090d7f 100644 --- a/container/containerd/handler.go +++ b/container/containerd/handler.go @@ -18,7 +18,6 @@ package containerd import ( "encoding/json" "fmt" - "path" "strings" "time" @@ -67,10 +66,7 @@ func newContainerdContainerHandler( includedMetrics container.MetricSet, ) (container.ContainerHandler, error) { // Create the cgroup paths. - cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints)) - for key, val := range cgroupSubsystems.MountPoints { - cgroupPaths[key] = path.Join(val, name) - } + cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems.MountPoints, name) // Generate the equivalent cgroup manager for this container. cgroupManager := &cgroupfs.Manager{ diff --git a/container/crio/factory.go b/container/crio/factory.go index 510b66f9..b3380b02 100644 --- a/container/crio/factory.go +++ b/container/crio/factory.go @@ -149,7 +149,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics // TODO determine crio version so we can work differently w/ future versions if needed - cgroupSubsystems, err := libcontainer.GetCgroupSubsystems() + cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics) if err != nil { return fmt.Errorf("failed to get cgroup subsystems: %v", err) } diff --git a/container/crio/handler.go b/container/crio/handler.go index d17ba6d9..4078474f 100644 --- a/container/crio/handler.go +++ b/container/crio/handler.go @@ -86,10 +86,7 @@ func newCrioContainerHandler( includedMetrics container.MetricSet, ) (container.ContainerHandler, error) { // Create the cgroup paths. - cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints)) - for key, val := range cgroupSubsystems.MountPoints { - cgroupPaths[key] = path.Join(val, name) - } + cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems.MountPoints, name) // Generate the equivalent cgroup manager for this container. cgroupManager := &cgroupfs.Manager{ diff --git a/container/docker/factory.go b/container/docker/factory.go index 7502781c..5802be03 100644 --- a/container/docker/factory.go +++ b/container/docker/factory.go @@ -325,7 +325,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics dockerAPIVersion, _ := APIVersion() - cgroupSubsystems, err := libcontainer.GetCgroupSubsystems() + cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics) if err != nil { return fmt.Errorf("failed to get cgroup subsystems: %v", err) } diff --git a/container/docker/handler.go b/container/docker/handler.go index 638350c0..ae8f48ef 100644 --- a/container/docker/handler.go +++ b/container/docker/handler.go @@ -134,10 +134,7 @@ func newDockerContainerHandler( zfsWatcher *zfs.ZfsWatcher, ) (container.ContainerHandler, error) { // Create the cgroup paths. - cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints)) - for key, val := range cgroupSubsystems.MountPoints { - cgroupPaths[key] = path.Join(val, name) - } + cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems.MountPoints, name) // Generate the equivalent cgroup manager for this container. cgroupManager := &cgroupfs.Manager{ diff --git a/container/libcontainer/handler.go b/container/libcontainer/handler.go index 5b44fb74..1951a1d6 100644 --- a/container/libcontainer/handler.go +++ b/container/libcontainer/handler.go @@ -66,8 +66,7 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) { libcontainerStats := &libcontainer.Stats{ CgroupStats: cgroupStats, } - withPerCPU := h.includedMetrics.Has(container.PerCpuUsageMetrics) - stats := newContainerStats(libcontainerStats, withPerCPU) + stats := newContainerStats(libcontainerStats, h.includedMetrics) if h.includedMetrics.Has(container.ProcessSchedulerMetrics) { pids, err := h.cgroupManager.GetAllPids() @@ -599,14 +598,16 @@ func setNetworkStats(libcontainerStats *libcontainer.Stats, ret *info.ContainerS } } -func newContainerStats(libcontainerStats *libcontainer.Stats, withPerCPU bool) *info.ContainerStats { +func newContainerStats(libcontainerStats *libcontainer.Stats, includedMetrics container.MetricSet) *info.ContainerStats { ret := &info.ContainerStats{ Timestamp: time.Now(), } if s := libcontainerStats.CgroupStats; s != nil { - setCpuStats(s, ret, withPerCPU) - setDiskIoStats(s, ret) + setCpuStats(s, ret, includedMetrics.Has(container.PerCpuUsageMetrics)) + if includedMetrics.Has(container.DiskIOMetrics) { + setDiskIoStats(s, ret) + } setMemoryStats(s, ret) } if len(libcontainerStats.Interfaces) > 0 { diff --git a/container/libcontainer/helpers.go b/container/libcontainer/helpers.go index 05554465..9877d75a 100644 --- a/container/libcontainer/helpers.go +++ b/container/libcontainer/helpers.go @@ -19,6 +19,7 @@ import ( info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/container" "github.com/opencontainers/runc/libcontainer/cgroups" "k8s.io/klog" ) @@ -33,18 +34,36 @@ type CgroupSubsystems struct { MountPoints map[string]string } -// Get information about the cgroup subsystems. -func GetCgroupSubsystems() (CgroupSubsystems, error) { +// Get information about the cgroup subsystems those we want +func GetCgroupSubsystems(includedMetrics container.MetricSet) (CgroupSubsystems, error) { // Get all cgroup mounts. allCgroups, err := cgroups.GetCgroupMounts(true) if err != nil { return CgroupSubsystems{}, err } - return getCgroupSubsystemsHelper(allCgroups) + disableCgroups := map[string]struct{}{} + + //currently we only support disable blkio subsystem + if !includedMetrics.Has(container.DiskIOMetrics) { + disableCgroups["blkio"] = struct{}{} + } + return getCgroupSubsystemsHelper(allCgroups, disableCgroups) } -func getCgroupSubsystemsHelper(allCgroups []cgroups.Mount) (CgroupSubsystems, error) { +// Get information about all the cgroup subsystems. +func GetAllCgroupSubsystems() (CgroupSubsystems, error) { + // Get all cgroup mounts. + allCgroups, err := cgroups.GetCgroupMounts(true) + if err != nil { + return CgroupSubsystems{}, err + } + + emptyDisableCgroups := map[string]struct{}{} + return getCgroupSubsystemsHelper(allCgroups, emptyDisableCgroups) +} + +func getCgroupSubsystemsHelper(allCgroups []cgroups.Mount, disableCgroups map[string]struct{}) (CgroupSubsystems, error) { if len(allCgroups) == 0 { return CgroupSubsystems{}, fmt.Errorf("failed to find cgroup mounts") } @@ -55,6 +74,9 @@ func getCgroupSubsystemsHelper(allCgroups []cgroups.Mount) (CgroupSubsystems, er mountPoints := make(map[string]string, len(allCgroups)) for _, mount := range allCgroups { for _, subsystem := range mount.Subsystems { + if _, exists := disableCgroups[subsystem]; exists { + continue + } if _, ok := supportedSubsystems[subsystem]; !ok { // Unsupported subsystem continue diff --git a/container/libcontainer/helpers_test.go b/container/libcontainer/helpers_test.go index bf6cf443..6b1f5bcc 100644 --- a/container/libcontainer/helpers_test.go +++ b/container/libcontainer/helpers_test.go @@ -97,7 +97,7 @@ func TestGetCgroupSubsystems(t *testing.T) { } for i, testCase := range testCases { - subSystems, err := getCgroupSubsystemsHelper(testCase.mounts) + subSystems, err := getCgroupSubsystemsHelper(testCase.mounts, map[string]struct{}{}) if testCase.err { if err == nil { t.Fatalf("[case %d] Expected error but didn't get one", i) diff --git a/container/mesos/factory.go b/container/mesos/factory.go index b2c61bb3..dd610a43 100644 --- a/container/mesos/factory.go +++ b/container/mesos/factory.go @@ -130,7 +130,7 @@ func Register( return fmt.Errorf("unable to create mesos agent client: %v", err) } - cgroupSubsystems, err := libcontainer.GetCgroupSubsystems() + cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics) if err != nil { return fmt.Errorf("failed to get cgroup subsystems: %v", err) } diff --git a/container/mesos/handler.go b/container/mesos/handler.go index 65d0b987..3ad9ef8d 100644 --- a/container/mesos/handler.go +++ b/container/mesos/handler.go @@ -17,7 +17,6 @@ package mesos import ( "fmt" - "path" "github.com/google/cadvisor/container" "github.com/google/cadvisor/container/common" @@ -68,9 +67,6 @@ func newMesosContainerHandler( client mesosAgentClient, ) (container.ContainerHandler, error) { cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems.MountPoints, name) - for key, val := range cgroupSubsystems.MountPoints { - cgroupPaths[key] = path.Join(val, name) - } // Generate the equivalent cgroup manager for this container. cgroupManager := &cgroupfs.Manager{ diff --git a/container/raw/factory.go b/container/raw/factory.go index afc852e3..7863da50 100644 --- a/container/raw/factory.go +++ b/container/raw/factory.go @@ -81,7 +81,7 @@ func (self *rawFactory) DebugInfo() map[string][]string { } func Register(machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics map[container.MetricKind]struct{}, rawPrefixWhiteList []string) error { - cgroupSubsystems, err := libcontainer.GetCgroupSubsystems() + cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics) if err != nil { return fmt.Errorf("failed to get cgroup subsystems: %v", err) } diff --git a/container/raw/handler.go b/container/raw/handler.go index 7e7504d9..c00e82d4 100644 --- a/container/raw/handler.go +++ b/container/raw/handler.go @@ -39,8 +39,9 @@ type rawContainerHandler struct { // (e.g.: "cpu" -> "/sys/fs/cgroup/cpu/test") cgroupPaths map[string]string - fsInfo fs.FsInfo - externalMounts []common.Mount + fsInfo fs.FsInfo + externalMounts []common.Mount + includedMetrics container.MetricSet libcontainerHandler *libcontainer.Handler } @@ -86,6 +87,7 @@ func newRawContainerHandler(name string, cgroupSubsystems *libcontainer.CgroupSu cgroupPaths: cgroupPaths, fsInfo: fsInfo, externalMounts: externalMounts, + includedMetrics: includedMetrics, libcontainerHandler: handler, }, nil } @@ -185,36 +187,40 @@ func fsToFsStats(fs *fs.Fs) info.FsStats { } func (self *rawContainerHandler) getFsStats(stats *info.ContainerStats) error { - var allFs []fs.Fs - // Get Filesystem information only for the root cgroup. - if isRootCgroup(self.name) { - filesystems, err := self.fsInfo.GetGlobalFsInfo() - if err != nil { - return err + var filesystems []fs.Fs + + if self.includedMetrics.Has(container.DiskUsageMetrics) || self.includedMetrics.Has(container.DiskIOMetrics) { + var err error + // Get Filesystem information only for the root cgroup. + if isRootCgroup(self.name) { + filesystems, err = self.fsInfo.GetGlobalFsInfo() + if err != nil { + return err + } + } else if len(self.externalMounts) > 0 { + var mountSet map[string]struct{} + mountSet = make(map[string]struct{}) + for _, mount := range self.externalMounts { + mountSet[mount.HostDir] = struct{}{} + } + filesystems, err = self.fsInfo.GetFsInfoForPath(mountSet) + if err != nil { + return err + } } - for i := range filesystems { - fs := filesystems[i] - stats.Filesystem = append(stats.Filesystem, fsToFsStats(&fs)) - } - allFs = filesystems - } else if len(self.externalMounts) > 0 { - var mountSet map[string]struct{} - mountSet = make(map[string]struct{}) - for _, mount := range self.externalMounts { - mountSet[mount.HostDir] = struct{}{} - } - filesystems, err := self.fsInfo.GetFsInfoForPath(mountSet) - if err != nil { - return err - } - for i := range filesystems { - fs := filesystems[i] - stats.Filesystem = append(stats.Filesystem, fsToFsStats(&fs)) - } - allFs = filesystems } - common.AssignDeviceNamesToDiskStats(&fsNamer{fs: allFs, factory: self.machineInfoFactory}, &stats.DiskIo) + if self.includedMetrics.Has(container.DiskUsageMetrics) { + for i := range filesystems { + fs := filesystems[i] + stats.Filesystem = append(stats.Filesystem, fsToFsStats(&fs)) + } + } + + if self.includedMetrics.Has(container.DiskIOMetrics) { + common.AssignDeviceNamesToDiskStats(&fsNamer{fs: filesystems, factory: self.machineInfoFactory}, &stats.DiskIo) + + } return nil } diff --git a/container/rkt/factory.go b/container/rkt/factory.go index cf806032..ab339708 100644 --- a/container/rkt/factory.go +++ b/container/rkt/factory.go @@ -78,7 +78,7 @@ func Register(machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, incl return fmt.Errorf("unable to get the RktPath variable %v", err) } - cgroupSubsystems, err := libcontainer.GetCgroupSubsystems() + cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics) if err != nil { return fmt.Errorf("failed to get cgroup subsystems: %v", err) } diff --git a/manager/watcher/raw/raw.go b/manager/watcher/raw/raw.go index 76983cbe..9dee46b6 100644 --- a/manager/watcher/raw/raw.go +++ b/manager/watcher/raw/raw.go @@ -45,7 +45,7 @@ type rawContainerWatcher struct { } func NewRawContainerWatcher() (watcher.ContainerWatcher, error) { - cgroupSubsystems, err := libcontainer.GetCgroupSubsystems() + cgroupSubsystems, err := libcontainer.GetAllCgroupSubsystems() if err != nil { return nil, fmt.Errorf("failed to get cgroup subsystems: %v", err) }