prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #267 from vishh/disk_usage

Browse Source 
Add filesystem usage support for docker containers.
This commit is contained in:
Victor Marmol 2014-10-14 16:33:22 +02:00
commit a2a3a92e4a
13 changed files with 241 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
container/docker/factory.go
View File

@ -39,6 +39,9 @@ type dockerFactory struct {
// Whether this system is using systemd.
useSystemd bool
// Whether docker is running with AUFS storage driver.
usesAufsDriver bool
client *docker.Client
}
@ -57,6 +60,7 @@ func (self *dockerFactory) NewContainerHandler(name string) (handler container.C
self.machineInfoFactory,
self.useSystemd,
*dockerRootDir,
self.usesAufsDriver,
)
return
}
@ -148,10 +152,19 @@ func Register(factory info.MachineInfoFactory) error {
return fmt.Errorf("Docker found, but not using native exec driver")
}
usesAufsDriver := false
for _, val := range *information {
if strings.Contains(val, "Driver=") && strings.Contains(val, "aufs") {
usesAufsDriver = true
break
}
}
f := &dockerFactory{
machineInfoFactory: factory,
useSystemd: systemd.UseSystemd(),
client: client,
usesAufsDriver: usesAufsDriver,
}
if f.useSystemd {
glog.Infof("System is using systemd")

80
container/docker/handler.go
View File

@ -25,11 +25,12 @@ import (
"github.com/docker/libcontainer"
"github.com/docker/libcontainer/cgroups"
"github.com/docker/libcontainer/cgroups/fs"
cgroup_fs "github.com/docker/libcontainer/cgroups/fs"
"github.com/fsouza/go-dockerclient"
"github.com/golang/glog"
"github.com/google/cadvisor/container"
containerLibcontainer "github.com/google/cadvisor/container/libcontainer"
"github.com/google/cadvisor/fs"
"github.com/google/cadvisor/info"
"github.com/google/cadvisor/utils"
)
@ -37,6 +38,11 @@ import (
// Relative path from Docker root to the libcontainer per-container state.
const pathToLibcontainerState = "execdriver/native"
// Path to aufs dir where all the files exist.
// aufs/layers is ignored here since it does not hold a lot of data.
// aufs/mnt contains the mount points used to compose the rootfs. Hence it is also ignored.
var pathToAufsDir = "aufs/diff"
var fileNotFound = errors.New("file not found")
type dockerContainerHandler struct {
@ -48,6 +54,9 @@ type dockerContainerHandler struct {
useSystemd bool
libcontainerStateDir string
cgroup cgroups.Cgroup
usesAufsDriver bool
fsInfo fs.FsInfo
storageDirs []string
}
func newDockerContainerHandler(
@ -56,7 +65,12 @@ func newDockerContainerHandler(
machineInfoFactory info.MachineInfoFactory,
useSystemd bool,
dockerRootDir string,
usesAufsDriver bool,
) (container.ContainerHandler, error) {
fsInfo, err := fs.NewFsInfo()
if err != nil {
return nil, err
}
handler := &dockerContainerHandler{
client: client,
name: name,
@ -67,7 +81,10 @@ func newDockerContainerHandler(
Parent: "/",
Name: name,
},
usesAufsDriver: usesAufsDriver,
fsInfo: fsInfo,
}
handler.storageDirs = append(handler.storageDirs, path.Join(dockerRootDir, pathToAufsDir, path.Base(name)))
if handler.isDockerRoot() {
return handler, nil
}
@ -213,9 +230,57 @@ func (self *dockerContainerHandler) GetSpec() (spec info.ContainerSpec, err erro
}
spec = libcontainerConfigToContainerSpec(libcontainerConfig, mi)
if self.usesAufsDriver {
spec.HasFilesystem = true
}
return
}
func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error {
// No support for non-aufs storage drivers.
if !self.usesAufsDriver {
return nil
}
// As of now we assume that all the storage dirs are on the same device.
// The first storage dir will be that of the image layers.
deviceInfo, err := self.fsInfo.GetDirFsDevice(self.storageDirs[0])
if err != nil {
return err
}
mi, err := self.machineInfoFactory.GetMachineInfo()
if err != nil {
return err
}
var limit uint64 = 0
// Docker does not impose any filesystem limits for containers. So use capacity as limit.
for _, fs := range mi.Filesystems {
if fs.Device == deviceInfo.Device {
limit = fs.Capacity
break
}
}
fsStat := info.FsStats{Device: deviceInfo.Device, Limit: limit}
var usage uint64 = 0
for _, dir := range self.storageDirs {
// TODO(Vishh): Add support for external mounts.
dirUsage, err := self.fsInfo.GetDirUsage(dir)
if err != nil {
return err
}
usage += dirUsage
}
fsStat.Usage = usage
stats.Filesystem = append(stats.Filesystem, fsStat)
return nil
}
func (self *dockerContainerHandler) GetStats() (stats *info.ContainerStats, err error) {
if self.isDockerRoot() {
return &info.ContainerStats{}, nil
@ -229,7 +294,16 @@ func (self *dockerContainerHandler) GetStats() (stats *info.ContainerStats, err
return
}
return containerLibcontainer.GetStats(&self.cgroup, state)
stats, err = containerLibcontainer.GetStats(&self.cgroup, state)
if err != nil {
return
}
err = self.getFsStats(stats)
if err != nil {
return
}
return stats, nil
}
func (self *dockerContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) {
@ -271,7 +345,7 @@ func (self *dockerContainerHandler) ListThreads(listType container.ListType) ([]
}
func (self *dockerContainerHandler) ListProcesses(listType container.ListType) ([]int, error) {
return fs.GetPids(&self.cgroup)
return cgroup_fs.GetPids(&self.cgroup)
}
func (self *dockerContainerHandler) WatchSubcontainers(events chan container.SubcontainerEvent) error {

5
container/raw/handler.go
View File

@ -162,10 +162,13 @@ func (self *rawContainerHandler) GetStats() (*info.ContainerStats, error) {
}
// Get Filesystem information only for the root cgroup.
if self.name == "/" {
stats.Filesystem, err = self.fsInfo.GetFsStats()
filesystems, err := self.fsInfo.GetGlobalFsInfo()
if err != nil {
return nil, err
}
for _, fs := range filesystems {
stats.Filesystem = append(stats.Filesystem, info.FsStats{fs.Device, fs.Capacity, fs.Capacity - fs.Free})
}
}
return stats, nil

61
fs/fs.go
View File

@ -10,22 +10,24 @@ package fs
import "C"
import (
"fmt"
"os/exec"
"strconv"
"strings"
"syscall"
"unsafe"
"github.com/docker/docker/pkg/mount"
"github.com/golang/glog"
"github.com/google/cadvisor/info"
)
type partition struct {
mountpoint string
major uint32
minor uint32
major uint
minor uint
}
type FsInfoImpl struct {
type RealFsInfo struct {
partitions map[string]partition
}
@ -43,31 +45,66 @@ func NewFsInfo() (FsInfo, error) {
if _, ok := partitions[mount.Source]; ok {
continue
}
partitions[mount.Source] = partition{mount.Mountpoint, uint32(mount.Major), uint32(mount.Minor)}
partitions[mount.Source] = partition{mount.Mountpoint, uint(mount.Major), uint(mount.Minor)}
}
return &FsInfoImpl{partitions}, nil
return &RealFsInfo{partitions}, nil
}
func (self *FsInfoImpl) GetFsStats() ([]info.FsStats, error) {
filesystems := make([]info.FsStats, 0)
func (self *RealFsInfo) GetGlobalFsInfo() ([]Fs, error) {
filesystems := make([]Fs, 0)
for device, partition := range self.partitions {
total, free, err := getVfsStats(partition.mountpoint)
if err != nil {
glog.Errorf("Statvfs failed. Error: %v", err)
} else {
fsStat := info.FsStats{
deviceInfo := DeviceInfo{
Device: device,
Major: uint(partition.major),
Minor: uint(partition.minor),
Capacity: total,
Free: free,
}
filesystems = append(filesystems, fsStat)
fs := Fs{deviceInfo, total, free}
filesystems = append(filesystems, fs)
}
}
return filesystems, nil
}
func major(devNumber uint64) uint {
return uint((devNumber >> 8) & 0xfff)
}
func minor(devNumber uint64) uint {
return uint((devNumber & 0xff) | ((devNumber >> 12) & 0xfff00))
}
func (self *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) {
var buf syscall.Stat_t
err := syscall.Stat(dir, &buf)
if err != nil {
return nil, fmt.Errorf("stat failed on %s with error: %s", dir, err)
}
major := major(buf.Dev)
minor := minor(buf.Dev)
for device, partition := range self.partitions {
if partition.major == major && partition.minor == minor {
return &DeviceInfo{device, major, minor}, nil
}
}
return nil, fmt.Errorf("could not find device with major: %d, minor: %d in cached partitions map", major, minor)
}
func (self *RealFsInfo) GetDirUsage(dir string) (uint64, error) {
out, err := exec.Command("du", "-s", dir).CombinedOutput()
if err != nil {
return 0, fmt.Errorf("du command failed on %s with output %s - %s", dir, out, err)
}
usageInKb, err := strconv.ParseUint(strings.Fields(string(out))[0], 10, 64)
if err != nil {
return 0, fmt.Errorf("cannot parse 'du' output %s - %s", out, err)
}
return usageInKb * 1024, nil
}
func getVfsStats(path string) (total uint64, free uint64, err error) {
_p0, err := syscall.BytePtrFromString(path)
if err != nil {

20
fs/types.go
View File

@ -1,8 +1,24 @@
package fs
import "github.com/google/cadvisor/info"
type DeviceInfo struct {
Device string
Major uint
Minor uint
}
type Fs struct {
DeviceInfo
Capacity uint64
Free uint64
}
type FsInfo interface {
// Returns capacity and free space, in bytes, of all the ext2, ext3, ext4 filesystems on the host.
GetFsStats() ([]info.FsStats, error)
GetGlobalFsInfo() ([]Fs, error)
// Returns number of bytes occupied by 'dir'.
GetDirUsage(dir string) (uint64, error)
// Returns the block device info of the filesystem on which 'dir' resides.
GetDirFsDevice(dir string) (*DeviceInfo, error)
}

11
info/container.go
View File

@ -232,11 +232,14 @@ type NetworkStats struct {
}
type FsStats struct {
// The block device name associated with the filesystem.
Device string `json:"device,omitempty"`
Major uint `json:"major"`
Minor uint `json:"minor"`
Capacity uint64 `json:"capacity"`
Free uint64 `json:"free"`
// Number of bytes that can be consumed by the container on this filesystem.
Limit uint64 `json:"capacity"`
// Number of bytes that is consumed by the container on this filesystem.
Usage uint64 `json:"usage"`
}
type ContainerStats struct {

11
info/machine.go
View File

@ -14,12 +14,23 @@
package info
type FsInfo struct {
// Block device associated with the filesystem.
Device string `json:"device"`
// Total number of bytes available on the filesystem.
Capacity uint64 `json:"capacity"`
}
type MachineInfo struct {
// The number of cores in this machine.
NumCores int `json:"num_cores"`
// The amount of memory (in bytes) in this machine
MemoryCapacity int64 `json:"memory_capacity"`
// Filesystems on this machine.
Filesystems []FsInfo `json:"filesystems"`
}
type VersionInfo struct {

19
manager/machine.go
View File

@ -25,6 +25,7 @@ import (
dclient "github.com/fsouza/go-dockerclient"
"github.com/google/cadvisor/container/docker"
"github.com/google/cadvisor/fs"
"github.com/google/cadvisor/info"
)
@ -59,10 +60,24 @@ func getMachineInfo() (*info.MachineInfo, error) {
// Capacity is in KB, convert it to bytes.
memoryCapacity = memoryCapacity * 1024
return &info.MachineInfo{
fsInfo, err := fs.NewFsInfo()
if err != nil {
return nil, err
}
filesystems, err := fsInfo.GetGlobalFsInfo()
if err != nil {
return nil, err
}
machineInfo := &info.MachineInfo{
NumCores: numCores,
MemoryCapacity: memoryCapacity,
}, nil
}
for _, fs := range filesystems {
machineInfo.Filesystems = append(machineInfo.Filesystems, info.FsInfo{fs.Device, fs.Capacity})
}
return machineInfo, nil
}
func getVersionInfo() (*info.VersionInfo, error) {

4
pages/containers.go
View File

@ -262,8 +262,8 @@ func getFsStats(stats []*info.ContainerStats) []info.FsStats {
return stats[len(stats)-1].Filesystem
}
func getFsUsagePercent(capacity, free uint64) uint64 {
return uint64((float64(capacity-free) / float64(capacity)) * 100)
func getFsUsagePercent(limit, used uint64) uint64 {
return uint64((float64(used) / float64(limit)) * 100)
}
func ServerContainersPage(m manager.Manager, w http.ResponseWriter, u *url.URL) error {

4
pages/containers_html.go
View File

@ -161,12 +161,12 @@ const containersHtmlTemplate = `
<div class="col-sm-9">
<div class="progress">
<div id="memory-usage-chart"></div>
<div class="progress-bar progress-bar-danger" style="width: {{getFsUsagePercent .Capacity .Free}}%">
<div class="progress-bar progress-bar-danger" style="width: {{getFsUsagePercent .Limit .Usage}}%">
</div>
</div>
</div>
<div class="col-sm-3">
{{printSize .Capacity}} {{printUnit .Capacity}} ({{getFsUsagePercent .Capacity .Free}}%)
{{printSize .Limit}} {{printUnit .Limit}} ({{getFsUsagePercent .Limit .Usage}}%)
</div>
{{end}}
{{end}}

34
storage/bigquery/bigquery.go
View File

@ -65,10 +65,10 @@ const (
colTxErrors string = "tx_errors"
// Filesystem device.
colFsDevice = "fs_device"
// Filesystem capacity.
colFsCapacity = "fs_capacity"
// Filesystem limit.
colFsLimit = "fs_limit"
// Filesystem available space.
colFsFree = "fs_free"
colFsUsage = "fs_usage"
)
// TODO(jnagal): Infer schema through reflection. (See bigquery/client/example)
@ -165,12 +165,12 @@ func (self *bigqueryStorage) GetSchema() *bigquery.TableSchema {
i++
fields[i] = &bigquery.TableFieldSchema{
Type: typeInteger,
Name: colFsCapacity,
Name: colFsLimit,
}
i++
fields[i] = &bigquery.TableFieldSchema{
Type: typeInteger,
Name: colFsFree,
Name: colFsUsage,
}
return &bigquery.TableSchema{
Fields: fields,
@ -243,8 +243,8 @@ func (self *bigqueryStorage) containerFilesystemStatsToRows(
for _, fsStat := range stats.Filesystem {
row := make(map[string]interface{}, 0)
row[colFsDevice] = fsStat.Device
row[colFsCapacity] = fsStat.Capacity
row[colFsFree] = fsStat.Free
row[colFsLimit] = fsStat.Limit
row[colFsUsage] = fsStat.Usage
rows = append(rows, row)
}
return rows
@ -354,25 +354,25 @@ func (self *bigqueryStorage) valuesToContainerStats(columns []string, values []i
} else {
stats.Filesystem[0].Device = device
}
case col == colFsCapacity:
capacity, err := convertToUint64(v)
case col == colFsLimit:
limit, err := convertToUint64(v)
if err != nil {
return nil, fmt.Errorf("filesystem capacity field %+v invalid: %s", v, err)
return nil, fmt.Errorf("filesystem limit field %+v invalid: %s", v, err)
}
if len(stats.Filesystem) == 0 {
stats.Filesystem = append(stats.Filesystem, info.FsStats{Capacity: capacity})
stats.Filesystem = append(stats.Filesystem, info.FsStats{Limit: limit})
} else {
stats.Filesystem[0].Capacity = capacity
stats.Filesystem[0].Limit = limit
}
case col == colFsFree:
free, err := convertToUint64(v)
case col == colFsUsage:
usage, err := convertToUint64(v)
if err != nil {
return nil, fmt.Errorf("filesystem free field %+v invalid: %s", v, err)
return nil, fmt.Errorf("filesystem usage field %+v invalid: %s", v, err)
}
if len(stats.Filesystem) == 0 {
stats.Filesystem = append(stats.Filesystem, info.FsStats{Free: free})
stats.Filesystem = append(stats.Filesystem, info.FsStats{Usage: usage})
} else {
stats.Filesystem[0].Free = free
stats.Filesystem[0].Usage = usage
}
}
if err != nil {

36
storage/influxdb/influxdb.go
View File

@ -53,10 +53,10 @@ const (
colTxErrors string = "tx_errors"
// Filesystem device.
colFsDevice = "fs_device"
// Filesystem capacity.
colFsCapacity = "fs_capacity"
// Filesystem available space.
colFsFree = "fs_free"
// Filesystem limit.
colFsLimit = "fs_limit"
// Filesystem usage.
colFsUsage = "fs_usage"
)
func (self *influxdbStorage) getSeriesDefaultValues(
@ -96,11 +96,11 @@ func (self *influxdbStorage) containerFilesystemStatsToSeries(
columns = append(columns, colFsDevice)
values = append(values, fsStat.Device)
columns = append(columns, colFsCapacity)
values = append(values, fsStat.Capacity)
columns = append(columns, colFsLimit)
values = append(values, fsStat.Limit)
columns = append(columns, colFsFree)
values = append(values, fsStat.Free)
columns = append(columns, colFsUsage)
values = append(values, fsStat.Usage)
series = append(series, self.newSeries(columns, values))
}
return series
@ -224,25 +224,25 @@ func (self *influxdbStorage) valuesToContainerStats(columns []string, values []i
} else {
stats.Filesystem[0].Device = device
}
case col == colFsCapacity:
capacity, err := convertToUint64(v)
case col == colFsLimit:
limit, err := convertToUint64(v)
if err != nil {
return nil, fmt.Errorf("filesystem capacity field %+v invalid: %s", v, err)
return nil, fmt.Errorf("filesystem limit field %+v invalid: %s", v, err)
}
if len(stats.Filesystem) == 0 {
stats.Filesystem = append(stats.Filesystem, info.FsStats{Capacity: capacity})
stats.Filesystem = append(stats.Filesystem, info.FsStats{Limit: limit})
} else {
stats.Filesystem[0].Capacity = capacity
stats.Filesystem[0].Limit = limit
}
case col == colFsFree:
free, err := convertToUint64(v)
case col == colFsUsage:
usage, err := convertToUint64(v)
if err != nil {
return nil, fmt.Errorf("filesystem free field %+v invalid: %s", v, err)
return nil, fmt.Errorf("filesystem usage field %+v invalid: %s", v, err)
}
if len(stats.Filesystem) == 0 {
stats.Filesystem = append(stats.Filesystem, info.FsStats{Free: free})
stats.Filesystem = append(stats.Filesystem, info.FsStats{Usage: usage})
} else {
stats.Filesystem[0].Free = free
stats.Filesystem[0].Usage = usage
}
}
if err != nil {

4
storage/test/storagetests.go
View File

@ -61,8 +61,8 @@ func buildTrace(cpu, mem []uint64, duration time.Duration) []*info.ContainerStat
stats.Filesystem = make([]info.FsStats, 1)
stats.Filesystem[0].Device = "/dev/sda1"
stats.Filesystem[0].Capacity = 1024000000
stats.Filesystem[0].Free = 1024000
stats.Filesystem[0].Limit = 1024000000
stats.Filesystem[0].Usage = 1024000
ret[i] = stats
}
return ret