Merge pull request #2103 from WanLinghao/disable-diskio

Add support to disable diskIO metrics

cadvisor.go

@@ -78,6 +78,7 @@ var (
 	// List of metrics that can be ignored.
 	ignoreWhitelist = container.MetricSet{
 		container.DiskUsageMetrics:        struct{}{},
+		container.DiskIOMetrics:           struct{}{},
 		container.NetworkUsageMetrics:     struct{}{},
 		container.NetworkTcpUsageMetrics:  struct{}{},
 		container.NetworkUdpUsageMetrics:  struct{}{},
@@ -115,7 +116,7 @@ func (ml *metricSetValue) Set(value string) error {
 }
 
 func init() {
-	flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'disk', 'network', 'tcp', 'udp', 'percpu', 'sched', 'process'. Note: tcp and udp are disabled by default due to high CPU usage.")
+	flag.Var(&ignoreMetrics, "disable_metrics", "comma-separated list of `metrics` to be disabled. Options are 'disk', 'diskIO', network', 'tcp', 'udp', 'percpu', 'sched', 'process'.")
 
 	// Default logging verbosity to V(2)
 	flag.Set("v", "2")

container/containerd/factory.go

@@ -128,7 +128,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics
 		return fmt.Errorf("failed to fetch containerd client version: %v", err)
 	}
 
-	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
+	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics)
 	if err != nil {
 		return fmt.Errorf("failed to get cgroup subsystems: %v", err)
 	}

container/containerd/handler.go

@@ -18,7 +18,6 @@ package containerd
 import (
 	"encoding/json"
 	"fmt"
-	"path"
 	"strings"
 	"time"
 
@@ -67,10 +66,7 @@ func newContainerdContainerHandler(
 	includedMetrics container.MetricSet,
 ) (container.ContainerHandler, error) {
 	// Create the cgroup paths.
-	cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints))
-	for key, val := range cgroupSubsystems.MountPoints {
-		cgroupPaths[key] = path.Join(val, name)
-	}
+	cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems.MountPoints, name)
 
 	// Generate the equivalent cgroup manager for this container.
 	cgroupManager := &cgroupfs.Manager{

container/crio/factory.go

@@ -149,7 +149,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics
 
 	// TODO determine crio version so we can work differently w/ future versions if needed
 
-	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
+	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics)
 	if err != nil {
 		return fmt.Errorf("failed to get cgroup subsystems: %v", err)
 	}

container/crio/handler.go

@@ -86,10 +86,7 @@ func newCrioContainerHandler(
 	includedMetrics container.MetricSet,
 ) (container.ContainerHandler, error) {
 	// Create the cgroup paths.
-	cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints))
-	for key, val := range cgroupSubsystems.MountPoints {
-		cgroupPaths[key] = path.Join(val, name)
-	}
+	cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems.MountPoints, name)
 
 	// Generate the equivalent cgroup manager for this container.
 	cgroupManager := &cgroupfs.Manager{

container/docker/factory.go

@@ -325,7 +325,7 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics
 
 	dockerAPIVersion, _ := APIVersion()
 
-	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
+	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics)
 	if err != nil {
 		return fmt.Errorf("failed to get cgroup subsystems: %v", err)
 	}

container/docker/handler.go

@@ -134,10 +134,7 @@ func newDockerContainerHandler(
 	zfsWatcher *zfs.ZfsWatcher,
 ) (container.ContainerHandler, error) {
 	// Create the cgroup paths.
-	cgroupPaths := make(map[string]string, len(cgroupSubsystems.MountPoints))
-	for key, val := range cgroupSubsystems.MountPoints {
-		cgroupPaths[key] = path.Join(val, name)
-	}
+	cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems.MountPoints, name)
 
 	// Generate the equivalent cgroup manager for this container.
 	cgroupManager := &cgroupfs.Manager{

container/libcontainer/handler.go

@@ -66,8 +66,7 @@ func (h *Handler) GetStats() (*info.ContainerStats, error) {
 	libcontainerStats := &libcontainer.Stats{
 		CgroupStats: cgroupStats,
 	}
-	withPerCPU := h.includedMetrics.Has(container.PerCpuUsageMetrics)
-	stats := newContainerStats(libcontainerStats, withPerCPU)
+	stats := newContainerStats(libcontainerStats, h.includedMetrics)
 
 	if h.includedMetrics.Has(container.ProcessSchedulerMetrics) {
 		pids, err := h.cgroupManager.GetAllPids()
@@ -599,14 +598,16 @@ func setNetworkStats(libcontainerStats *libcontainer.Stats, ret *info.ContainerS
 	}
 }
 
-func newContainerStats(libcontainerStats *libcontainer.Stats, withPerCPU bool) *info.ContainerStats {
+func newContainerStats(libcontainerStats *libcontainer.Stats, includedMetrics container.MetricSet) *info.ContainerStats {
 	ret := &info.ContainerStats{
 		Timestamp: time.Now(),
 	}
 
 	if s := libcontainerStats.CgroupStats; s != nil {
-		setCpuStats(s, ret, withPerCPU)
+		setCpuStats(s, ret, includedMetrics.Has(container.PerCpuUsageMetrics))
+		if includedMetrics.Has(container.DiskIOMetrics) {
 			setDiskIoStats(s, ret)
+		}
 		setMemoryStats(s, ret)
 	}
 	if len(libcontainerStats.Interfaces) > 0 {

container/libcontainer/helpers.go

@@ -19,6 +19,7 @@ import (
 
 	info "github.com/google/cadvisor/info/v1"
 
+	"github.com/google/cadvisor/container"
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"k8s.io/klog"
 )
@@ -33,18 +34,36 @@ type CgroupSubsystems struct {
 	MountPoints map[string]string
 }
 
-// Get information about the cgroup subsystems.
-func GetCgroupSubsystems() (CgroupSubsystems, error) {
+// Get information about the cgroup subsystems those we want
+func GetCgroupSubsystems(includedMetrics container.MetricSet) (CgroupSubsystems, error) {
 	// Get all cgroup mounts.
 	allCgroups, err := cgroups.GetCgroupMounts(true)
 	if err != nil {
 		return CgroupSubsystems{}, err
 	}
 
-	return getCgroupSubsystemsHelper(allCgroups)
+	disableCgroups := map[string]struct{}{}
+
+	//currently we only support disable blkio subsystem
+	if !includedMetrics.Has(container.DiskIOMetrics) {
+		disableCgroups["blkio"] = struct{}{}
+	}
+	return getCgroupSubsystemsHelper(allCgroups, disableCgroups)
 }
 
-func getCgroupSubsystemsHelper(allCgroups []cgroups.Mount) (CgroupSubsystems, error) {
+// Get information about all the cgroup subsystems.
+func GetAllCgroupSubsystems() (CgroupSubsystems, error) {
+	// Get all cgroup mounts.
+	allCgroups, err := cgroups.GetCgroupMounts(true)
+	if err != nil {
+		return CgroupSubsystems{}, err
+	}
+
+	emptyDisableCgroups := map[string]struct{}{}
+	return getCgroupSubsystemsHelper(allCgroups, emptyDisableCgroups)
+}
+
+func getCgroupSubsystemsHelper(allCgroups []cgroups.Mount, disableCgroups map[string]struct{}) (CgroupSubsystems, error) {
 	if len(allCgroups) == 0 {
 		return CgroupSubsystems{}, fmt.Errorf("failed to find cgroup mounts")
 	}
@@ -55,6 +74,9 @@ func getCgroupSubsystemsHelper(allCgroups []cgroups.Mount) (CgroupSubsystems, er
 	mountPoints := make(map[string]string, len(allCgroups))
 	for _, mount := range allCgroups {
 		for _, subsystem := range mount.Subsystems {
+			if _, exists := disableCgroups[subsystem]; exists {
+				continue
+			}
 			if _, ok := supportedSubsystems[subsystem]; !ok {
 				// Unsupported subsystem
 				continue

container/libcontainer/helpers_test.go

@@ -97,7 +97,7 @@ func TestGetCgroupSubsystems(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		subSystems, err := getCgroupSubsystemsHelper(testCase.mounts)
+		subSystems, err := getCgroupSubsystemsHelper(testCase.mounts, map[string]struct{}{})
 		if testCase.err {
 			if err == nil {
 				t.Fatalf("[case %d] Expected error but didn't get one", i)

container/mesos/factory.go

@@ -130,7 +130,7 @@ func Register(
 		return fmt.Errorf("unable to create mesos agent client: %v", err)
 	}
 
-	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
+	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics)
 	if err != nil {
 		return fmt.Errorf("failed to get cgroup subsystems: %v", err)
 	}

container/mesos/handler.go

@@ -17,7 +17,6 @@ package mesos
 
 import (
 	"fmt"
-	"path"
 
 	"github.com/google/cadvisor/container"
 	"github.com/google/cadvisor/container/common"
@@ -68,9 +67,6 @@ func newMesosContainerHandler(
 	client mesosAgentClient,
 ) (container.ContainerHandler, error) {
 	cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems.MountPoints, name)
-	for key, val := range cgroupSubsystems.MountPoints {
-		cgroupPaths[key] = path.Join(val, name)
-	}
 
 	// Generate the equivalent cgroup manager for this container.
 	cgroupManager := &cgroupfs.Manager{

container/raw/factory.go

@@ -81,7 +81,7 @@ func (self *rawFactory) DebugInfo() map[string][]string {
 }
 
 func Register(machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics map[container.MetricKind]struct{}, rawPrefixWhiteList []string) error {
-	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
+	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics)
 	if err != nil {
 		return fmt.Errorf("failed to get cgroup subsystems: %v", err)
 	}

container/raw/handler.go

@@ -41,6 +41,7 @@ type rawContainerHandler struct {
 
 	fsInfo          fs.FsInfo
 	externalMounts  []common.Mount
+	includedMetrics container.MetricSet
 
 	libcontainerHandler *libcontainer.Handler
 }
@@ -86,6 +87,7 @@ func newRawContainerHandler(name string, cgroupSubsystems *libcontainer.CgroupSu
 		cgroupPaths:         cgroupPaths,
 		fsInfo:              fsInfo,
 		externalMounts:      externalMounts,
+		includedMetrics:     includedMetrics,
 		libcontainerHandler: handler,
 	}, nil
 }
@@ -185,36 +187,40 @@ func fsToFsStats(fs *fs.Fs) info.FsStats {
 }
 
 func (self *rawContainerHandler) getFsStats(stats *info.ContainerStats) error {
-	var allFs []fs.Fs
+	var filesystems []fs.Fs
+
+	if self.includedMetrics.Has(container.DiskUsageMetrics) || self.includedMetrics.Has(container.DiskIOMetrics) {
+		var err error
 		// Get Filesystem information only for the root cgroup.
 		if isRootCgroup(self.name) {
-		filesystems, err := self.fsInfo.GetGlobalFsInfo()
+			filesystems, err = self.fsInfo.GetGlobalFsInfo()
 			if err != nil {
 				return err
 			}
-		for i := range filesystems {
-			fs := filesystems[i]
-			stats.Filesystem = append(stats.Filesystem, fsToFsStats(&fs))
-		}
-		allFs = filesystems
 		} else if len(self.externalMounts) > 0 {
 			var mountSet map[string]struct{}
 			mountSet = make(map[string]struct{})
 			for _, mount := range self.externalMounts {
 				mountSet[mount.HostDir] = struct{}{}
 			}
-		filesystems, err := self.fsInfo.GetFsInfoForPath(mountSet)
+			filesystems, err = self.fsInfo.GetFsInfoForPath(mountSet)
 			if err != nil {
 				return err
 			}
+		}
+	}
+
+	if self.includedMetrics.Has(container.DiskUsageMetrics) {
 		for i := range filesystems {
 			fs := filesystems[i]
 			stats.Filesystem = append(stats.Filesystem, fsToFsStats(&fs))
 		}
-		allFs = filesystems
 	}
 
-	common.AssignDeviceNamesToDiskStats(&fsNamer{fs: allFs, factory: self.machineInfoFactory}, &stats.DiskIo)
+	if self.includedMetrics.Has(container.DiskIOMetrics) {
+		common.AssignDeviceNamesToDiskStats(&fsNamer{fs: filesystems, factory: self.machineInfoFactory}, &stats.DiskIo)
+
+	}
 	return nil
 }
 

container/rkt/factory.go

@@ -78,7 +78,7 @@ func Register(machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, incl
 		return fmt.Errorf("unable to get the RktPath variable %v", err)
 	}
 
-	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
+	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems(includedMetrics)
 	if err != nil {
 		return fmt.Errorf("failed to get cgroup subsystems: %v", err)
 	}

manager/watcher/raw/raw.go

@@ -45,7 +45,7 @@ type rawContainerWatcher struct {
 }
 
 func NewRawContainerWatcher() (watcher.ContainerWatcher, error) {
-	cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
+	cgroupSubsystems, err := libcontainer.GetAllCgroupSubsystems()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get cgroup subsystems: %v", err)
 	}