Merge pull request #267 from vishh/disk_usage

Add filesystem usage support for docker containers.

container/docker/factory.go

@@ -39,6 +39,9 @@ type dockerFactory struct {
 	// Whether this system is using systemd.
 	useSystemd bool
 
+	// Whether docker is running with AUFS storage driver.
+	usesAufsDriver bool
+
 	client *docker.Client
 }
 
@@ -57,6 +60,7 @@ func (self *dockerFactory) NewContainerHandler(name string) (handler container.C
 		self.machineInfoFactory,
 		self.useSystemd,
 		*dockerRootDir,
+		self.usesAufsDriver,
 	)
 	return
 }
@@ -148,10 +152,19 @@ func Register(factory info.MachineInfoFactory) error {
 		return fmt.Errorf("Docker found, but not using native exec driver")
 	}
 
+	usesAufsDriver := false
+	for _, val := range *information {
+		if strings.Contains(val, "Driver=") && strings.Contains(val, "aufs") {
+			usesAufsDriver = true
+			break
+		}
+	}
+
 	f := &dockerFactory{
 		machineInfoFactory: factory,
 		useSystemd:         systemd.UseSystemd(),
 		client:             client,
+		usesAufsDriver:     usesAufsDriver,
 	}
 	if f.useSystemd {
 		glog.Infof("System is using systemd")

container/docker/handler.go

@@ -25,11 +25,12 @@ import (
 
 	"github.com/docker/libcontainer"
 	"github.com/docker/libcontainer/cgroups"
-	"github.com/docker/libcontainer/cgroups/fs"
+	cgroup_fs "github.com/docker/libcontainer/cgroups/fs"
 	"github.com/fsouza/go-dockerclient"
 	"github.com/golang/glog"
 	"github.com/google/cadvisor/container"
 	containerLibcontainer "github.com/google/cadvisor/container/libcontainer"
+	"github.com/google/cadvisor/fs"
 	"github.com/google/cadvisor/info"
 	"github.com/google/cadvisor/utils"
 )
@@ -37,6 +38,11 @@ import (
 // Relative path from Docker root to the libcontainer per-container state.
 const pathToLibcontainerState = "execdriver/native"
 
+// Path to aufs dir where all the files exist.
+// aufs/layers is ignored here since it does not hold a lot of data.
+// aufs/mnt contains the mount points used to compose the rootfs. Hence it is also ignored.
+var pathToAufsDir = "aufs/diff"
+
 var fileNotFound = errors.New("file not found")
 
 type dockerContainerHandler struct {
@@ -48,6 +54,9 @@ type dockerContainerHandler struct {
 	useSystemd           bool
 	libcontainerStateDir string
 	cgroup               cgroups.Cgroup
+	usesAufsDriver       bool
+	fsInfo               fs.FsInfo
+	storageDirs          []string
 }
 
 func newDockerContainerHandler(
@@ -56,7 +65,12 @@ func newDockerContainerHandler(
 	machineInfoFactory info.MachineInfoFactory,
 	useSystemd bool,
 	dockerRootDir string,
+	usesAufsDriver bool,
 ) (container.ContainerHandler, error) {
+	fsInfo, err := fs.NewFsInfo()
+	if err != nil {
+		return nil, err
+	}
 	handler := &dockerContainerHandler{
 		client:               client,
 		name:                 name,
@@ -67,7 +81,10 @@ func newDockerContainerHandler(
 			Parent: "/",
 			Name:   name,
 		},
+		usesAufsDriver: usesAufsDriver,
+		fsInfo:         fsInfo,
 	}
+	handler.storageDirs = append(handler.storageDirs, path.Join(dockerRootDir, pathToAufsDir, path.Base(name)))
 	if handler.isDockerRoot() {
 		return handler, nil
 	}
@@ -213,9 +230,57 @@ func (self *dockerContainerHandler) GetSpec() (spec info.ContainerSpec, err erro
 	}
 
 	spec = libcontainerConfigToContainerSpec(libcontainerConfig, mi)
+
+	if self.usesAufsDriver {
+		spec.HasFilesystem = true
+	}
+
 	return
 }
 
+func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error {
+	// No support for non-aufs storage drivers.
+	if !self.usesAufsDriver {
+		return nil
+	}
+
+	// As of now we assume that all the storage dirs are on the same device.
+	// The first storage dir will be that of the image layers.
+	deviceInfo, err := self.fsInfo.GetDirFsDevice(self.storageDirs[0])
+	if err != nil {
+		return err
+	}
+
+	mi, err := self.machineInfoFactory.GetMachineInfo()
+	if err != nil {
+		return err
+	}
+	var limit uint64 = 0
+	// Docker does not impose any filesystem limits for containers. So use capacity as limit.
+	for _, fs := range mi.Filesystems {
+		if fs.Device == deviceInfo.Device {
+			limit = fs.Capacity
+			break
+		}
+	}
+
+	fsStat := info.FsStats{Device: deviceInfo.Device, Limit: limit}
+
+	var usage uint64 = 0
+	for _, dir := range self.storageDirs {
+		// TODO(Vishh): Add support for external mounts.
+		dirUsage, err := self.fsInfo.GetDirUsage(dir)
+		if err != nil {
+			return err
+		}
+		usage += dirUsage
+	}
+	fsStat.Usage = usage
+	stats.Filesystem = append(stats.Filesystem, fsStat)
+
+	return nil
+}
+
 func (self *dockerContainerHandler) GetStats() (stats *info.ContainerStats, err error) {
 	if self.isDockerRoot() {
 		return &info.ContainerStats{}, nil
@@ -229,7 +294,16 @@ func (self *dockerContainerHandler) GetStats() (stats *info.ContainerStats, err
 		return
 	}
 
-	return containerLibcontainer.GetStats(&self.cgroup, state)
+	stats, err = containerLibcontainer.GetStats(&self.cgroup, state)
+	if err != nil {
+		return
+	}
+	err = self.getFsStats(stats)
+	if err != nil {
+		return
+	}
+
+	return stats, nil
 }
 
 func (self *dockerContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) {
@@ -271,7 +345,7 @@ func (self *dockerContainerHandler) ListThreads(listType container.ListType) ([]
 }
 
 func (self *dockerContainerHandler) ListProcesses(listType container.ListType) ([]int, error) {
-	return fs.GetPids(&self.cgroup)
+	return cgroup_fs.GetPids(&self.cgroup)
 }
 
 func (self *dockerContainerHandler) WatchSubcontainers(events chan container.SubcontainerEvent) error {

container/raw/handler.go

@@ -162,10 +162,13 @@ func (self *rawContainerHandler) GetStats() (*info.ContainerStats, error) {
 	}
 	// Get Filesystem information only for the root cgroup.
 	if self.name == "/" {
-		stats.Filesystem, err = self.fsInfo.GetFsStats()
+		filesystems, err := self.fsInfo.GetGlobalFsInfo()
 		if err != nil {
 			return nil, err
 		}
+		for _, fs := range filesystems {
+			stats.Filesystem = append(stats.Filesystem, info.FsStats{fs.Device, fs.Capacity, fs.Capacity - fs.Free})
+		}
 	}
 
 	return stats, nil

fs/fs.go

@@ -10,22 +10,24 @@ package fs
 import "C"
 
 import (
+	"fmt"
+	"os/exec"
+	"strconv"
 	"strings"
 	"syscall"
 	"unsafe"
 
 	"github.com/docker/docker/pkg/mount"
 	"github.com/golang/glog"
-	"github.com/google/cadvisor/info"
 )
 
 type partition struct {
 	mountpoint string
-	major      uint32
+	major      uint
-	minor      uint32
+	minor      uint
 }
 
-type FsInfoImpl struct {
+type RealFsInfo struct {
 	partitions map[string]partition
 }
 
@@ -43,31 +45,66 @@ func NewFsInfo() (FsInfo, error) {
 		if _, ok := partitions[mount.Source]; ok {
 			continue
 		}
-		partitions[mount.Source] = partition{mount.Mountpoint, uint32(mount.Major), uint32(mount.Minor)}
+		partitions[mount.Source] = partition{mount.Mountpoint, uint(mount.Major), uint(mount.Minor)}
 	}
-	return &FsInfoImpl{partitions}, nil
+	return &RealFsInfo{partitions}, nil
 }
 
-func (self *FsInfoImpl) GetFsStats() ([]info.FsStats, error) {
+func (self *RealFsInfo) GetGlobalFsInfo() ([]Fs, error) {
-	filesystems := make([]info.FsStats, 0)
+	filesystems := make([]Fs, 0)
 	for device, partition := range self.partitions {
 		total, free, err := getVfsStats(partition.mountpoint)
 		if err != nil {
 			glog.Errorf("Statvfs failed. Error: %v", err)
 		} else {
-			fsStat := info.FsStats{
+			deviceInfo := DeviceInfo{
-				Device:   device,
+				Device: device,
-				Major:    uint(partition.major),
+				Major:  uint(partition.major),
-				Minor:    uint(partition.minor),
+				Minor:  uint(partition.minor),
-				Capacity: total,
-				Free:     free,
 			}
-			filesystems = append(filesystems, fsStat)
+			fs := Fs{deviceInfo, total, free}
+			filesystems = append(filesystems, fs)
 		}
 	}
 	return filesystems, nil
 }
 
+func major(devNumber uint64) uint {
+	return uint((devNumber >> 8) & 0xfff)
+}
+
+func minor(devNumber uint64) uint {
+	return uint((devNumber & 0xff) | ((devNumber >> 12) & 0xfff00))
+}
+
+func (self *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) {
+	var buf syscall.Stat_t
+	err := syscall.Stat(dir, &buf)
+	if err != nil {
+		return nil, fmt.Errorf("stat failed on %s with error: %s", dir, err)
+	}
+	major := major(buf.Dev)
+	minor := minor(buf.Dev)
+	for device, partition := range self.partitions {
+		if partition.major == major && partition.minor == minor {
+			return &DeviceInfo{device, major, minor}, nil
+		}
+	}
+	return nil, fmt.Errorf("could not find device with major: %d, minor: %d in cached partitions map", major, minor)
+}
+
+func (self *RealFsInfo) GetDirUsage(dir string) (uint64, error) {
+	out, err := exec.Command("du", "-s", dir).CombinedOutput()
+	if err != nil {
+		return 0, fmt.Errorf("du command failed on %s with output %s - %s", dir, out, err)
+	}
+	usageInKb, err := strconv.ParseUint(strings.Fields(string(out))[0], 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("cannot parse 'du' output %s - %s", out, err)
+	}
+	return usageInKb * 1024, nil
+}
+
 func getVfsStats(path string) (total uint64, free uint64, err error) {
 	_p0, err := syscall.BytePtrFromString(path)
 	if err != nil {

fs/types.go

@@ -1,8 +1,24 @@
 package fs
 
-import "github.com/google/cadvisor/info"
+type DeviceInfo struct {
+	Device string
+	Major  uint
+	Minor  uint
+}
+
+type Fs struct {
+	DeviceInfo
+	Capacity uint64
+	Free     uint64
+}
 
 type FsInfo interface {
 	// Returns capacity and free space, in bytes, of all the ext2, ext3, ext4 filesystems on the host.
-	GetFsStats() ([]info.FsStats, error)
+	GetGlobalFsInfo() ([]Fs, error)
+
+	// Returns number of bytes occupied by 'dir'.
+	GetDirUsage(dir string) (uint64, error)
+
+	// Returns the block device info of the filesystem on which 'dir' resides.
+	GetDirFsDevice(dir string) (*DeviceInfo, error)
 }

info/container.go

@@ -232,11 +232,14 @@ type NetworkStats struct {
 }
 
 type FsStats struct {
-	Device   string `json:"device,omitempty"`
+	// The block device name associated with the filesystem.
-	Major    uint   `json:"major"`
+	Device string `json:"device,omitempty"`
-	Minor    uint   `json:"minor"`
+
-	Capacity uint64 `json:"capacity"`
+	// Number of bytes that can be consumed by the container on this filesystem.
-	Free     uint64 `json:"free"`
+	Limit uint64 `json:"capacity"`
+
+	// Number of bytes that is consumed by the container on this filesystem.
+	Usage uint64 `json:"usage"`
 }
 
 type ContainerStats struct {

info/machine.go

@@ -14,12 +14,23 @@
 
 package info
 
+type FsInfo struct {
+	// Block device associated with the filesystem.
+	Device string `json:"device"`
+
+	// Total number of bytes available on the filesystem.
+	Capacity uint64 `json:"capacity"`
+}
+
 type MachineInfo struct {
 	// The number of cores in this machine.
 	NumCores int `json:"num_cores"`
 
 	// The amount of memory (in bytes) in this machine
 	MemoryCapacity int64 `json:"memory_capacity"`
+
+	// Filesystems on this machine.
+	Filesystems []FsInfo `json:"filesystems"`
 }
 
 type VersionInfo struct {

manager/machine.go

@@ -25,6 +25,7 @@ import (
 
 	dclient "github.com/fsouza/go-dockerclient"
 	"github.com/google/cadvisor/container/docker"
+	"github.com/google/cadvisor/fs"
 	"github.com/google/cadvisor/info"
 )
 
@@ -59,10 +60,24 @@ func getMachineInfo() (*info.MachineInfo, error) {
 	// Capacity is in KB, convert it to bytes.
 	memoryCapacity = memoryCapacity * 1024
 
-	return &info.MachineInfo{
+	fsInfo, err := fs.NewFsInfo()
+	if err != nil {
+		return nil, err
+	}
+	filesystems, err := fsInfo.GetGlobalFsInfo()
+	if err != nil {
+		return nil, err
+	}
+
+	machineInfo := &info.MachineInfo{
 		NumCores:       numCores,
 		MemoryCapacity: memoryCapacity,
-	}, nil
+	}
+	for _, fs := range filesystems {
+		machineInfo.Filesystems = append(machineInfo.Filesystems, info.FsInfo{fs.Device, fs.Capacity})
+	}
+
+	return machineInfo, nil
 }
 
 func getVersionInfo() (*info.VersionInfo, error) {

pages/containers.go

@@ -262,8 +262,8 @@ func getFsStats(stats []*info.ContainerStats) []info.FsStats {
 	return stats[len(stats)-1].Filesystem
 }
 
-func getFsUsagePercent(capacity, free uint64) uint64 {
+func getFsUsagePercent(limit, used uint64) uint64 {
-	return uint64((float64(capacity-free) / float64(capacity)) * 100)
+	return uint64((float64(used) / float64(limit)) * 100)
 }
 
 func ServerContainersPage(m manager.Manager, w http.ResponseWriter, u *url.URL) error {

pages/containers_html.go

@@ -161,12 +161,12 @@ const containersHtmlTemplate = `
             <div class="col-sm-9">
 	      <div class="progress">
 	        <div id="memory-usage-chart"></div>
-                <div class="progress-bar progress-bar-danger" style="width: {{getFsUsagePercent .Capacity .Free}}%">
+                <div class="progress-bar progress-bar-danger" style="width: {{getFsUsagePercent .Limit .Usage}}%">
                 </div>
               </div>
             </div>
             <div class="col-sm-3"> 
-	      {{printSize .Capacity}} {{printUnit .Capacity}} ({{getFsUsagePercent .Capacity .Free}}%)
+	      {{printSize .Limit}} {{printUnit .Limit}} ({{getFsUsagePercent .Limit .Usage}}%)
 	    </div>
 	    {{end}}
 	    {{end}}

storage/bigquery/bigquery.go

@@ -65,10 +65,10 @@ const (
 	colTxErrors string = "tx_errors"
 	// Filesystem device.
 	colFsDevice = "fs_device"
-	// Filesystem capacity.
+	// Filesystem limit.
-	colFsCapacity = "fs_capacity"
+	colFsLimit = "fs_limit"
 	// Filesystem available space.
-	colFsFree = "fs_free"
+	colFsUsage = "fs_usage"
 )
 
 // TODO(jnagal): Infer schema through reflection. (See bigquery/client/example)
@@ -165,12 +165,12 @@ func (self *bigqueryStorage) GetSchema() *bigquery.TableSchema {
 	i++
 	fields[i] = &bigquery.TableFieldSchema{
 		Type: typeInteger,
-		Name: colFsCapacity,
+		Name: colFsLimit,
 	}
 	i++
 	fields[i] = &bigquery.TableFieldSchema{
 		Type: typeInteger,
-		Name: colFsFree,
+		Name: colFsUsage,
 	}
 	return &bigquery.TableSchema{
 		Fields: fields,
@@ -243,8 +243,8 @@ func (self *bigqueryStorage) containerFilesystemStatsToRows(
 	for _, fsStat := range stats.Filesystem {
 		row := make(map[string]interface{}, 0)
 		row[colFsDevice] = fsStat.Device
-		row[colFsCapacity] = fsStat.Capacity
+		row[colFsLimit] = fsStat.Limit
-		row[colFsFree] = fsStat.Free
+		row[colFsUsage] = fsStat.Usage
 		rows = append(rows, row)
 	}
 	return rows
@@ -354,25 +354,25 @@ func (self *bigqueryStorage) valuesToContainerStats(columns []string, values []i
 			} else {
 				stats.Filesystem[0].Device = device
 			}
-		case col == colFsCapacity:
+		case col == colFsLimit:
-			capacity, err := convertToUint64(v)
+			limit, err := convertToUint64(v)
 			if err != nil {
-				return nil, fmt.Errorf("filesystem capacity field %+v invalid: %s", v, err)
+				return nil, fmt.Errorf("filesystem limit field %+v invalid: %s", v, err)
 			}
 			if len(stats.Filesystem) == 0 {
-				stats.Filesystem = append(stats.Filesystem, info.FsStats{Capacity: capacity})
+				stats.Filesystem = append(stats.Filesystem, info.FsStats{Limit: limit})
 			} else {
-				stats.Filesystem[0].Capacity = capacity
+				stats.Filesystem[0].Limit = limit
 			}
-		case col == colFsFree:
+		case col == colFsUsage:
-			free, err := convertToUint64(v)
+			usage, err := convertToUint64(v)
 			if err != nil {
-				return nil, fmt.Errorf("filesystem free field %+v invalid: %s", v, err)
+				return nil, fmt.Errorf("filesystem usage field %+v invalid: %s", v, err)
 			}
 			if len(stats.Filesystem) == 0 {
-				stats.Filesystem = append(stats.Filesystem, info.FsStats{Free: free})
+				stats.Filesystem = append(stats.Filesystem, info.FsStats{Usage: usage})
 			} else {
-				stats.Filesystem[0].Free = free
+				stats.Filesystem[0].Usage = usage
 			}
 		}
 		if err != nil {

storage/influxdb/influxdb.go

@@ -53,10 +53,10 @@ const (
 	colTxErrors string = "tx_errors"
 	// Filesystem device.
 	colFsDevice = "fs_device"
-	// Filesystem capacity.
+	// Filesystem limit.
-	colFsCapacity = "fs_capacity"
+	colFsLimit = "fs_limit"
-	// Filesystem available space.
+	// Filesystem usage.
-	colFsFree = "fs_free"
+	colFsUsage = "fs_usage"
 )
 
 func (self *influxdbStorage) getSeriesDefaultValues(
@@ -96,11 +96,11 @@ func (self *influxdbStorage) containerFilesystemStatsToSeries(
 		columns = append(columns, colFsDevice)
 		values = append(values, fsStat.Device)
 
-		columns = append(columns, colFsCapacity)
+		columns = append(columns, colFsLimit)
-		values = append(values, fsStat.Capacity)
+		values = append(values, fsStat.Limit)
 
-		columns = append(columns, colFsFree)
+		columns = append(columns, colFsUsage)
-		values = append(values, fsStat.Free)
+		values = append(values, fsStat.Usage)
 		series = append(series, self.newSeries(columns, values))
 	}
 	return series
@@ -224,25 +224,25 @@ func (self *influxdbStorage) valuesToContainerStats(columns []string, values []i
 			} else {
 				stats.Filesystem[0].Device = device
 			}
-		case col == colFsCapacity:
+		case col == colFsLimit:
-			capacity, err := convertToUint64(v)
+			limit, err := convertToUint64(v)
 			if err != nil {
-				return nil, fmt.Errorf("filesystem capacity field %+v invalid: %s", v, err)
+				return nil, fmt.Errorf("filesystem limit field %+v invalid: %s", v, err)
 			}
 			if len(stats.Filesystem) == 0 {
-				stats.Filesystem = append(stats.Filesystem, info.FsStats{Capacity: capacity})
+				stats.Filesystem = append(stats.Filesystem, info.FsStats{Limit: limit})
 			} else {
-				stats.Filesystem[0].Capacity = capacity
+				stats.Filesystem[0].Limit = limit
 			}
-		case col == colFsFree:
+		case col == colFsUsage:
-			free, err := convertToUint64(v)
+			usage, err := convertToUint64(v)
 			if err != nil {
-				return nil, fmt.Errorf("filesystem free field %+v invalid: %s", v, err)
+				return nil, fmt.Errorf("filesystem usage field %+v invalid: %s", v, err)
 			}
 			if len(stats.Filesystem) == 0 {
-				stats.Filesystem = append(stats.Filesystem, info.FsStats{Free: free})
+				stats.Filesystem = append(stats.Filesystem, info.FsStats{Usage: usage})
 			} else {
-				stats.Filesystem[0].Free = free
+				stats.Filesystem[0].Usage = usage
 			}
 		}
 		if err != nil {

storage/test/storagetests.go

@@ -61,8 +61,8 @@ func buildTrace(cpu, mem []uint64, duration time.Duration) []*info.ContainerStat
 
 		stats.Filesystem = make([]info.FsStats, 1)
 		stats.Filesystem[0].Device = "/dev/sda1"
-		stats.Filesystem[0].Capacity = 1024000000
+		stats.Filesystem[0].Limit = 1024000000
-		stats.Filesystem[0].Free = 1024000
+		stats.Filesystem[0].Usage = 1024000
 		ret[i] = stats
 	}
 	return ret