From 5e8fecea6a5b9a147fc20ba14a31a9e2ca63ec7a Mon Sep 17 00:00:00 2001 From: Vishnu Kannan Date: Mon, 6 Oct 2014 09:49:59 +0000 Subject: [PATCH 1/2] Adding filesystem usage calculation for docker containers. This patch also includes some internal refactoring. 'machine' api now exports a list of all existing 'ext*' filesystems along with the capacity. --- container/docker/handler.go | 70 ++++++++++++++++++++++++++++++++++-- container/raw/handler.go | 5 ++- fs/fs.go | 67 ++++++++++++++++++++++++++-------- fs/types.go | 20 +++++++++-- info/container.go | 13 ++++--- info/machine.go | 11 ++++++ manager/machine.go | 19 ++++++++-- pages/containers.go | 4 +-- pages/containers_html.go | 4 +-- storage/bigquery/bigquery.go | 34 +++++++++--------- storage/influxdb/influxdb.go | 36 +++++++++---------- 11 files changed, 216 insertions(+), 67 deletions(-) diff --git a/container/docker/handler.go b/container/docker/handler.go index 958e0080..733c13bc 100644 --- a/container/docker/handler.go +++ b/container/docker/handler.go @@ -25,11 +25,12 @@ import ( "github.com/docker/libcontainer" "github.com/docker/libcontainer/cgroups" - "github.com/docker/libcontainer/cgroups/fs" + cgroup_fs "github.com/docker/libcontainer/cgroups/fs" "github.com/fsouza/go-dockerclient" "github.com/golang/glog" "github.com/google/cadvisor/container" containerLibcontainer "github.com/google/cadvisor/container/libcontainer" + "github.com/google/cadvisor/fs" "github.com/google/cadvisor/info" "github.com/google/cadvisor/utils" ) @@ -37,6 +38,11 @@ import ( // Relative path from Docker root to the libcontainer per-container state. const pathToLibcontainerState = "execdriver/native" +// Path to aufs dir where all the files exist. +// aufs/layers is ignored here since it does not hold a lot of data. +// aufs/mnt contains the mount points used to compose the rootfs. Hence it is also ignored. +var pathToAufsDir = "aufs/diff" + var fileNotFound = errors.New("file not found") type dockerContainerHandler struct { @@ -48,6 +54,8 @@ type dockerContainerHandler struct { useSystemd bool libcontainerStateDir string cgroup cgroups.Cgroup + fsInfo fs.FsInfo + storageDirs []string } func newDockerContainerHandler( @@ -57,6 +65,10 @@ func newDockerContainerHandler( useSystemd bool, dockerRootDir string, ) (container.ContainerHandler, error) { + fsInfo, err := fs.NewFsInfo() + if err != nil { + return nil, err + } handler := &dockerContainerHandler{ client: client, name: name, @@ -67,7 +79,9 @@ func newDockerContainerHandler( Parent: "/", Name: name, }, + fsInfo: fsInfo, } + handler.storageDirs = append(handler.storageDirs, path.Join(dockerRootDir, pathToAufsDir, path.Base(name))) if handler.isDockerRoot() { return handler, nil } @@ -213,9 +227,50 @@ func (self *dockerContainerHandler) GetSpec() (spec info.ContainerSpec, err erro } spec = libcontainerConfigToContainerSpec(libcontainerConfig, mi) + + spec.HasFilesystem = true + return } +func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error { + // As of now we assume that all the storage dirs are on the same device. + // The first storage dir will be that of the image layers. + deviceInfo, err := self.fsInfo.GetDirFsDevice(self.storageDirs[0]) + if err != nil { + return err + } + + mi, err := self.machineInfoFactory.GetMachineInfo() + if err != nil { + return err + } + var limit uint64 = 0 + // Docker does not impose any filesystem limits for containers. So use capacity as limit. + for _, fs := range mi.Filesystems { + if fs.Device == deviceInfo.Device { + limit = fs.Capacity + break + } + } + + fsStat := info.FsStats{Device: deviceInfo.Device, Limit: limit} + + var usage uint64 = 0 + for _, dir := range self.storageDirs { + // TODO(Vishh): Add support for external mounts. + dirUsage, err := self.fsInfo.GetDirUsage(dir) + if err != nil { + return err + } + usage += dirUsage + } + fsStat.Usage = usage + stats.Filesystem = append(stats.Filesystem, fsStat) + + return nil +} + func (self *dockerContainerHandler) GetStats() (stats *info.ContainerStats, err error) { if self.isDockerRoot() { return &info.ContainerStats{}, nil @@ -229,7 +284,16 @@ func (self *dockerContainerHandler) GetStats() (stats *info.ContainerStats, err return } - return containerLibcontainer.GetStats(&self.cgroup, state) + stats, err = containerLibcontainer.GetStats(&self.cgroup, state) + if err != nil { + return + } + err = self.getFsStats(stats) + if err != nil { + return + } + + return stats, nil } func (self *dockerContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) { @@ -271,7 +335,7 @@ func (self *dockerContainerHandler) ListThreads(listType container.ListType) ([] } func (self *dockerContainerHandler) ListProcesses(listType container.ListType) ([]int, error) { - return fs.GetPids(&self.cgroup) + return cgroup_fs.GetPids(&self.cgroup) } func (self *dockerContainerHandler) WatchSubcontainers(events chan container.SubcontainerEvent) error { diff --git a/container/raw/handler.go b/container/raw/handler.go index 6441f652..9a3c521c 100644 --- a/container/raw/handler.go +++ b/container/raw/handler.go @@ -162,10 +162,13 @@ func (self *rawContainerHandler) GetStats() (*info.ContainerStats, error) { } // Get Filesystem information only for the root cgroup. if self.name == "/" { - stats.Filesystem, err = self.fsInfo.GetFsStats() + filesystems, err := self.fsInfo.GetGlobalFsInfo() if err != nil { return nil, err } + for _, fs := range filesystems { + stats.Filesystem = append(stats.Filesystem, info.FsStats{fs.Device, fs.Capacity, fs.Capacity - fs.Free}) + } } return stats, nil diff --git a/fs/fs.go b/fs/fs.go index bf07959e..b93b519a 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -10,22 +10,24 @@ package fs import "C" import ( + "fmt" + "os/exec" + "strconv" "strings" "syscall" "unsafe" "github.com/docker/docker/pkg/mount" "github.com/golang/glog" - "github.com/google/cadvisor/info" ) type partition struct { mountpoint string - major uint32 - minor uint32 + major uint + minor uint } -type FsInfoImpl struct { +type RealFsInfo struct { partitions map[string]partition } @@ -43,31 +45,66 @@ func NewFsInfo() (FsInfo, error) { if _, ok := partitions[mount.Source]; ok { continue } - partitions[mount.Source] = partition{mount.Mountpoint, uint32(mount.Major), uint32(mount.Minor)} + partitions[mount.Source] = partition{mount.Mountpoint, uint(mount.Major), uint(mount.Minor)} } - return &FsInfoImpl{partitions}, nil + return &RealFsInfo{partitions}, nil } -func (self *FsInfoImpl) GetFsStats() ([]info.FsStats, error) { - filesystems := make([]info.FsStats, 0) +func (self *RealFsInfo) GetGlobalFsInfo() ([]Fs, error) { + filesystems := make([]Fs, 0) for device, partition := range self.partitions { total, free, err := getVfsStats(partition.mountpoint) if err != nil { glog.Errorf("Statvfs failed. Error: %v", err) } else { - fsStat := info.FsStats{ - Device: device, - Major: uint(partition.major), - Minor: uint(partition.minor), - Capacity: total, - Free: free, + deviceInfo := DeviceInfo{ + Device: device, + Major: uint(partition.major), + Minor: uint(partition.minor), } - filesystems = append(filesystems, fsStat) + fs := Fs{deviceInfo, total, free} + filesystems = append(filesystems, fs) } } return filesystems, nil } +func major(devNumber uint64) uint { + return uint((devNumber >> 8) & 0xfff) +} + +func minor(devNumber uint64) uint { + return uint((devNumber & 0xff) | ((devNumber >> 12) & 0xfff00)) +} + +func (self *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) { + var buf syscall.Stat_t + err := syscall.Stat(dir, &buf) + if err != nil { + return nil, fmt.Errorf("stat failed on %s with error: %s", dir, err) + } + major := major(buf.Dev) + minor := minor(buf.Dev) + for device, partition := range self.partitions { + if partition.major == major && partition.minor == minor { + return &DeviceInfo{device, major, minor}, nil + } + } + return nil, fmt.Errorf("could not find device with major: %d, minor: %d in cached partitions map", major, minor) +} + +func (self *RealFsInfo) GetDirUsage(dir string) (uint64, error) { + out, err := exec.Command("du", "-s", dir).CombinedOutput() + if err != nil { + return 0, fmt.Errorf("du command failed on %s with output %s - %s", dir, out, err) + } + usageInKb, err := strconv.ParseUint(strings.Fields(string(out))[0], 10, 64) + if err != nil { + return 0, fmt.Errorf("cannot parse 'du' output %s - %s", out, err) + } + return usageInKb * 1024, nil +} + func getVfsStats(path string) (total uint64, free uint64, err error) { _p0, err := syscall.BytePtrFromString(path) if err != nil { diff --git a/fs/types.go b/fs/types.go index 8cc78877..14592aff 100644 --- a/fs/types.go +++ b/fs/types.go @@ -1,8 +1,24 @@ package fs -import "github.com/google/cadvisor/info" +type DeviceInfo struct { + Device string + Major uint + Minor uint +} + +type Fs struct { + DeviceInfo + Capacity uint64 + Free uint64 +} type FsInfo interface { // Returns capacity and free space, in bytes, of all the ext2, ext3, ext4 filesystems on the host. - GetFsStats() ([]info.FsStats, error) + GetGlobalFsInfo() ([]Fs, error) + + // Returns number of bytes occupied by 'dir'. + GetDirUsage(dir string) (uint64, error) + + // Returns the block device info of the filesystem on which 'dir' resides. + GetDirFsDevice(dir string) (*DeviceInfo, error) } diff --git a/info/container.go b/info/container.go index 28cd1118..d99394f7 100644 --- a/info/container.go +++ b/info/container.go @@ -232,11 +232,14 @@ type NetworkStats struct { } type FsStats struct { - Device string `json:"device,omitempty"` - Major uint `json:"major"` - Minor uint `json:"minor"` - Capacity uint64 `json:"capacity"` - Free uint64 `json:"free"` + // The block device name associated with the filesystem. + Device string `json:"device,omitempty"` + + // Number of bytes that can be consumed by the container on this filesystem. + Limit uint64 `json:"capacity"` + + // Number of bytes that is consumed by the container on this filesystem. + Usage uint64 `json:"free"` } type ContainerStats struct { diff --git a/info/machine.go b/info/machine.go index 7415dc9e..0e73a8e0 100644 --- a/info/machine.go +++ b/info/machine.go @@ -14,12 +14,23 @@ package info +type FsInfo struct { + // Block device associated with the filesystem. + Device string `json:"device"` + + // Total number of bytes available on the filesystem. + Capacity uint64 `json:"capacity"` +} + type MachineInfo struct { // The number of cores in this machine. NumCores int `json:"num_cores"` // The amount of memory (in bytes) in this machine MemoryCapacity int64 `json:"memory_capacity"` + + // Filesystems on this machine. + Filesystems []FsInfo `json:"filesystems"` } type VersionInfo struct { diff --git a/manager/machine.go b/manager/machine.go index 3ca4fd2f..8cf05f0d 100644 --- a/manager/machine.go +++ b/manager/machine.go @@ -25,6 +25,7 @@ import ( dclient "github.com/fsouza/go-dockerclient" "github.com/google/cadvisor/container/docker" + "github.com/google/cadvisor/fs" "github.com/google/cadvisor/info" ) @@ -59,10 +60,24 @@ func getMachineInfo() (*info.MachineInfo, error) { // Capacity is in KB, convert it to bytes. memoryCapacity = memoryCapacity * 1024 - return &info.MachineInfo{ + fsInfo, err := fs.NewFsInfo() + if err != nil { + return nil, err + } + filesystems, err := fsInfo.GetGlobalFsInfo() + if err != nil { + return nil, err + } + + machineInfo := &info.MachineInfo{ NumCores: numCores, MemoryCapacity: memoryCapacity, - }, nil + } + for _, fs := range filesystems { + machineInfo.Filesystems = append(machineInfo.Filesystems, info.FsInfo{fs.Device, fs.Capacity}) + } + + return machineInfo, nil } func getVersionInfo() (*info.VersionInfo, error) { diff --git a/pages/containers.go b/pages/containers.go index 640e7725..d6ceb04c 100644 --- a/pages/containers.go +++ b/pages/containers.go @@ -262,8 +262,8 @@ func getFsStats(stats []*info.ContainerStats) []info.FsStats { return stats[len(stats)-1].Filesystem } -func getFsUsagePercent(capacity, free uint64) uint64 { - return uint64((float64(capacity-free) / float64(capacity)) * 100) +func getFsUsagePercent(limit, used uint64) uint64 { + return uint64((float64(used) / float64(limit)) * 100) } func ServerContainersPage(m manager.Manager, w http.ResponseWriter, u *url.URL) error { diff --git a/pages/containers_html.go b/pages/containers_html.go index 71c87bf2..d3d0979d 100644 --- a/pages/containers_html.go +++ b/pages/containers_html.go @@ -161,12 +161,12 @@ const containersHtmlTemplate = `
-
+
- {{printSize .Capacity}} {{printUnit .Capacity}} ({{getFsUsagePercent .Capacity .Free}}%) + {{printSize .Limit}} {{printUnit .Limit}} ({{getFsUsagePercent .Limit .Usage}}%)
{{end}} {{end}} diff --git a/storage/bigquery/bigquery.go b/storage/bigquery/bigquery.go index e57dafa8..7d344391 100644 --- a/storage/bigquery/bigquery.go +++ b/storage/bigquery/bigquery.go @@ -65,10 +65,10 @@ const ( colTxErrors string = "tx_errors" // Filesystem device. colFsDevice = "fs_device" - // Filesystem capacity. - colFsCapacity = "fs_capacity" + // Filesystem limit. + colFsLimit = "fs_limit" // Filesystem available space. - colFsFree = "fs_free" + colFsUsage = "fs_usage" ) // TODO(jnagal): Infer schema through reflection. (See bigquery/client/example) @@ -165,12 +165,12 @@ func (self *bigqueryStorage) GetSchema() *bigquery.TableSchema { i++ fields[i] = &bigquery.TableFieldSchema{ Type: typeInteger, - Name: colFsCapacity, + Name: colFsLimit, } i++ fields[i] = &bigquery.TableFieldSchema{ Type: typeInteger, - Name: colFsFree, + Name: colFsUsage, } return &bigquery.TableSchema{ Fields: fields, @@ -243,8 +243,8 @@ func (self *bigqueryStorage) containerFilesystemStatsToRows( for _, fsStat := range stats.Filesystem { row := make(map[string]interface{}, 0) row[colFsDevice] = fsStat.Device - row[colFsCapacity] = fsStat.Capacity - row[colFsFree] = fsStat.Free + row[colFsLimit] = fsStat.Limit + row[colFsUsage] = fsStat.Usage rows = append(rows, row) } return rows @@ -354,25 +354,25 @@ func (self *bigqueryStorage) valuesToContainerStats(columns []string, values []i } else { stats.Filesystem[0].Device = device } - case col == colFsCapacity: - capacity, err := convertToUint64(v) + case col == colFsLimit: + limit, err := convertToUint64(v) if err != nil { - return nil, fmt.Errorf("filesystem capacity field %+v invalid: %s", v, err) + return nil, fmt.Errorf("filesystem limit field %+v invalid: %s", v, err) } if len(stats.Filesystem) == 0 { - stats.Filesystem = append(stats.Filesystem, info.FsStats{Capacity: capacity}) + stats.Filesystem = append(stats.Filesystem, info.FsStats{Limit: limit}) } else { - stats.Filesystem[0].Capacity = capacity + stats.Filesystem[0].Limit = limit } - case col == colFsFree: - free, err := convertToUint64(v) + case col == colFsUsage: + usage, err := convertToUint64(v) if err != nil { - return nil, fmt.Errorf("filesystem free field %+v invalid: %s", v, err) + return nil, fmt.Errorf("filesystem usage field %+v invalid: %s", v, err) } if len(stats.Filesystem) == 0 { - stats.Filesystem = append(stats.Filesystem, info.FsStats{Free: free}) + stats.Filesystem = append(stats.Filesystem, info.FsStats{Usage: usage}) } else { - stats.Filesystem[0].Free = free + stats.Filesystem[0].Usage = usage } } if err != nil { diff --git a/storage/influxdb/influxdb.go b/storage/influxdb/influxdb.go index fc604e72..0374032d 100644 --- a/storage/influxdb/influxdb.go +++ b/storage/influxdb/influxdb.go @@ -53,10 +53,10 @@ const ( colTxErrors string = "tx_errors" // Filesystem device. colFsDevice = "fs_device" - // Filesystem capacity. - colFsCapacity = "fs_capacity" - // Filesystem available space. - colFsFree = "fs_free" + // Filesystem limit. + colFsLimit = "fs_limit" + // Filesystem usage. + colFsUsage = "fs_usage" ) func (self *influxdbStorage) getSeriesDefaultValues( @@ -96,11 +96,11 @@ func (self *influxdbStorage) containerFilesystemStatsToSeries( columns = append(columns, colFsDevice) values = append(values, fsStat.Device) - columns = append(columns, colFsCapacity) - values = append(values, fsStat.Capacity) + columns = append(columns, colFsLimit) + values = append(values, fsStat.Limit) - columns = append(columns, colFsFree) - values = append(values, fsStat.Free) + columns = append(columns, colFsUsage) + values = append(values, fsStat.Usage) series = append(series, self.newSeries(columns, values)) } return series @@ -224,25 +224,25 @@ func (self *influxdbStorage) valuesToContainerStats(columns []string, values []i } else { stats.Filesystem[0].Device = device } - case col == colFsCapacity: - capacity, err := convertToUint64(v) + case col == colFsLimit: + limit, err := convertToUint64(v) if err != nil { - return nil, fmt.Errorf("filesystem capacity field %+v invalid: %s", v, err) + return nil, fmt.Errorf("filesystem limit field %+v invalid: %s", v, err) } if len(stats.Filesystem) == 0 { - stats.Filesystem = append(stats.Filesystem, info.FsStats{Capacity: capacity}) + stats.Filesystem = append(stats.Filesystem, info.FsStats{Limit: limit}) } else { - stats.Filesystem[0].Capacity = capacity + stats.Filesystem[0].Limit = limit } - case col == colFsFree: - free, err := convertToUint64(v) + case col == colFsUsage: + usage, err := convertToUint64(v) if err != nil { - return nil, fmt.Errorf("filesystem free field %+v invalid: %s", v, err) + return nil, fmt.Errorf("filesystem usage field %+v invalid: %s", v, err) } if len(stats.Filesystem) == 0 { - stats.Filesystem = append(stats.Filesystem, info.FsStats{Free: free}) + stats.Filesystem = append(stats.Filesystem, info.FsStats{Usage: usage}) } else { - stats.Filesystem[0].Free = free + stats.Filesystem[0].Usage = usage } } if err != nil { From 0699e7029d095e1d8d2d89e65a719ab8e56ee295 Mon Sep 17 00:00:00 2001 From: Vishnu Kannan Date: Tue, 7 Oct 2014 08:34:59 +0000 Subject: [PATCH 2/2] Avoid storage usagge calculations when aufs driver is not being used. --- container/docker/factory.go | 13 +++++++++++++ container/docker/handler.go | 14 ++++++++++++-- info/container.go | 2 +- storage/test/storagetests.go | 4 ++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/container/docker/factory.go b/container/docker/factory.go index 68888096..dc916885 100644 --- a/container/docker/factory.go +++ b/container/docker/factory.go @@ -39,6 +39,9 @@ type dockerFactory struct { // Whether this system is using systemd. useSystemd bool + // Whether docker is running with AUFS storage driver. + usesAufsDriver bool + client *docker.Client } @@ -57,6 +60,7 @@ func (self *dockerFactory) NewContainerHandler(name string) (handler container.C self.machineInfoFactory, self.useSystemd, *dockerRootDir, + self.usesAufsDriver, ) return } @@ -148,10 +152,19 @@ func Register(factory info.MachineInfoFactory) error { return fmt.Errorf("Docker found, but not using native exec driver") } + usesAufsDriver := false + for _, val := range *information { + if strings.Contains(val, "Driver=") && strings.Contains(val, "aufs") { + usesAufsDriver = true + break + } + } + f := &dockerFactory{ machineInfoFactory: factory, useSystemd: systemd.UseSystemd(), client: client, + usesAufsDriver: usesAufsDriver, } if f.useSystemd { glog.Infof("System is using systemd") diff --git a/container/docker/handler.go b/container/docker/handler.go index 733c13bc..b7208e83 100644 --- a/container/docker/handler.go +++ b/container/docker/handler.go @@ -54,6 +54,7 @@ type dockerContainerHandler struct { useSystemd bool libcontainerStateDir string cgroup cgroups.Cgroup + usesAufsDriver bool fsInfo fs.FsInfo storageDirs []string } @@ -64,6 +65,7 @@ func newDockerContainerHandler( machineInfoFactory info.MachineInfoFactory, useSystemd bool, dockerRootDir string, + usesAufsDriver bool, ) (container.ContainerHandler, error) { fsInfo, err := fs.NewFsInfo() if err != nil { @@ -79,7 +81,8 @@ func newDockerContainerHandler( Parent: "/", Name: name, }, - fsInfo: fsInfo, + usesAufsDriver: usesAufsDriver, + fsInfo: fsInfo, } handler.storageDirs = append(handler.storageDirs, path.Join(dockerRootDir, pathToAufsDir, path.Base(name))) if handler.isDockerRoot() { @@ -228,12 +231,19 @@ func (self *dockerContainerHandler) GetSpec() (spec info.ContainerSpec, err erro spec = libcontainerConfigToContainerSpec(libcontainerConfig, mi) - spec.HasFilesystem = true + if self.usesAufsDriver { + spec.HasFilesystem = true + } return } func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error { + // No support for non-aufs storage drivers. + if !self.usesAufsDriver { + return nil + } + // As of now we assume that all the storage dirs are on the same device. // The first storage dir will be that of the image layers. deviceInfo, err := self.fsInfo.GetDirFsDevice(self.storageDirs[0]) diff --git a/info/container.go b/info/container.go index d99394f7..5eaacb59 100644 --- a/info/container.go +++ b/info/container.go @@ -239,7 +239,7 @@ type FsStats struct { Limit uint64 `json:"capacity"` // Number of bytes that is consumed by the container on this filesystem. - Usage uint64 `json:"free"` + Usage uint64 `json:"usage"` } type ContainerStats struct { diff --git a/storage/test/storagetests.go b/storage/test/storagetests.go index 6aeff3bc..128688a6 100644 --- a/storage/test/storagetests.go +++ b/storage/test/storagetests.go @@ -61,8 +61,8 @@ func buildTrace(cpu, mem []uint64, duration time.Duration) []*info.ContainerStat stats.Filesystem = make([]info.FsStats, 1) stats.Filesystem[0].Device = "/dev/sda1" - stats.Filesystem[0].Capacity = 1024000000 - stats.Filesystem[0].Free = 1024000 + stats.Filesystem[0].Limit = 1024000000 + stats.Filesystem[0].Usage = 1024000 ret[i] = stats } return ret