Merge pull request #1035 from vishh/log-usage

Expose container's write layer usage separately, with API refactoring

Makefile

@@ -18,7 +18,10 @@ all: format build test
 
 test:
 	@echo ">> running tests"
-	@$(GO) test -short $(pkgs)
+	@$(GO) test -short -race $(pkgs)
+
+test-integration: test
+	@./build/integration.sh
 
 format:
 	@echo ">> formatting code"

api/versions.go

@@ -31,6 +31,7 @@ const (
 	containersApi    = "containers"
 	subcontainersApi = "subcontainers"
 	machineApi       = "machine"
+	machineStatsApi  = "machinestats"
 	dockerApi        = "docker"
 	summaryApi       = "summary"
 	statsApi         = "stats"
@@ -62,8 +63,9 @@ func getApiVersions() []ApiVersion {
 	v1_2 := newVersion1_2(v1_1)
 	v1_3 := newVersion1_3(v1_2)
 	v2_0 := newVersion2_0()
+	v2_1 := newVersion2_1(v2_0)
 
-	return []ApiVersion{v1_0, v1_1, v1_2, v1_3, v2_0}
+	return []ApiVersion{v1_0, v1_1, v1_2, v1_3, v2_0, v2_1}
 
 }
 
@@ -357,13 +359,13 @@ func (self *version2_0) HandleRequest(requestType string, request []string, m ma
 	case statsApi:
 		name := getContainerName(request)
 		glog.V(4).Infof("Api - Stats: Looking for stats for container %q, options %+v", name, opt)
-		infos, err := m.GetContainerInfoV2(name, opt)
+		infos, err := m.GetRequestedContainersInfo(name, opt)
 		if err != nil {
 			return err
 		}
-		contStats := make(map[string][]*v2.ContainerStats, 0)
+		contStats := make(map[string][]v2.DeprecatedContainerStats, 0)
 		for name, cinfo := range infos {
-			contStats[name] = cinfo.Stats
+			contStats[name] = v2.DeprecatedStatsFromV1(cinfo)
 		}
 		return writeResult(contStats, w)
 	case customMetricsApi:
@@ -377,26 +379,27 @@ func (self *version2_0) HandleRequest(requestType string, request []string, m ma
 		for _, cinfo := range infos {
 			metrics := make(map[string]map[string][]info.MetricValBasic, 0)
 			for _, contStat := range cinfo.Stats {
-				if contStat.HasCustomMetrics {
+				if len(contStat.CustomMetrics) == 0 {
-					for name, allLabels := range contStat.CustomMetrics {
+					continue
-						metricLabels := make(map[string][]info.MetricValBasic, 0)
+				}
-						for _, metric := range allLabels {
+				for name, allLabels := range contStat.CustomMetrics {
-							if !metric.Timestamp.IsZero() {
+					metricLabels := make(map[string][]info.MetricValBasic, 0)
-								metVal := info.MetricValBasic{
+					for _, metric := range allLabels {
-									Timestamp:  metric.Timestamp,
+						if !metric.Timestamp.IsZero() {
-									IntValue:   metric.IntValue,
+							metVal := info.MetricValBasic{
-									FloatValue: metric.FloatValue,
+								Timestamp:  metric.Timestamp,
-								}
+								IntValue:   metric.IntValue,
-								labels := metrics[name]
+								FloatValue: metric.FloatValue,
-								if labels != nil {
+							}
-									values := labels[metric.Label]
+							labels := metrics[name]
-									values = append(values, metVal)
+							if labels != nil {
-									labels[metric.Label] = values
+								values := labels[metric.Label]
-									metrics[name] = labels
+								values = append(values, metVal)
-								} else {
+								labels[metric.Label] = values
-									metricLabels[metric.Label] = []info.MetricValBasic{metVal}
+								metrics[name] = labels
-									metrics[name] = metricLabels
+							} else {
-								}
+								metricLabels[metric.Label] = []info.MetricValBasic{metVal}
+								metrics[name] = metricLabels
 							}
 						}
 					}
@@ -449,6 +452,56 @@ func (self *version2_0) HandleRequest(requestType string, request []string, m ma
 	}
 }
 
+type version2_1 struct {
+	baseVersion *version2_0
+}
+
+func newVersion2_1(v *version2_0) *version2_1 {
+	return &version2_1{
+		baseVersion: v,
+	}
+}
+
+func (self *version2_1) Version() string {
+	return "v2.1"
+}
+
+func (self *version2_1) SupportedRequestTypes() []string {
+	return self.baseVersion.SupportedRequestTypes()
+}
+
+func (self *version2_1) HandleRequest(requestType string, request []string, m manager.Manager, w http.ResponseWriter, r *http.Request) error {
+	// Get the query request.
+	opt, err := getRequestOptions(r)
+	if err != nil {
+		return err
+	}
+
+	switch requestType {
+	case machineStatsApi:
+		glog.V(4).Infof("Api - MachineStats(%v)", request)
+		cont, err := m.GetRequestedContainersInfo("/", opt)
+		if err != nil {
+			return err
+		}
+		return writeResult(v2.MachineStatsFromV1(cont["/"]), w)
+	case statsApi:
+		name := getContainerName(request)
+		glog.V(4).Infof("Api - Stats: Looking for stats for container %q, options %+v", name, opt)
+		conts, err := m.GetRequestedContainersInfo(name, opt)
+		if err != nil {
+			return err
+		}
+		contStats := make(map[string][]*v2.ContainerStats, len(conts))
+		for name, cont := range conts {
+			contStats[name] = v2.ContainerStatsFromV1(&cont.Spec, cont.Stats)
+		}
+		return writeResult(contStats, w)
+	default:
+		return self.baseVersion.HandleRequest(requestType, request, m, w, r)
+	}
+}
+
 func getRequestOptions(r *http.Request) (v2.RequestOptions, error) {
 	supportedTypes := map[string]bool{
 		v2.TypeName:   true,

build/integration.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo ">> starting cAdvisor locally"
+sudo ./cadvisor &
+echo ">> running integration tests against local cAdvisor"
+godep go test github.com/google/cadvisor/integration/tests/... --vmodule=*=2
+if [ $? -ne 0 ]
+then
+    echo "Integration tests failed"
+fi
+echo ">> stopping cAdvisor"
+sudo pkill -9 cadvisor
+

client/client.go

@@ -25,7 +25,7 @@ import (
 	"path"
 	"strings"
 
-	info "github.com/google/cadvisor/info/v1"
+	"github.com/google/cadvisor/info/v1"
 
 	"github.com/golang/glog"
 )
@@ -35,7 +35,7 @@ type Client struct {
 	baseUrl string
 }
 
-// NewClient returns a new client with the specified base URL.
+// NewClient returns a new v1.3 client with the specified base URL.
 func NewClient(url string) (*Client, error) {
 	if !strings.HasSuffix(url, "/") {
 		url += "/"
@@ -47,9 +47,9 @@ func NewClient(url string) (*Client, error) {
 }
 
 // Returns all past events that satisfy the request
-func (self *Client) EventStaticInfo(name string) (einfo []*info.Event, err error) {
+func (self *Client) EventStaticInfo(name string) (einfo []*v1.Event, err error) {
 	u := self.eventsInfoUrl(name)
-	ret := new([]*info.Event)
+	ret := new([]*v1.Event)
 	if err = self.httpGetJsonData(ret, nil, u, "event info"); err != nil {
 		return
 	}
@@ -59,7 +59,7 @@ func (self *Client) EventStaticInfo(name string) (einfo []*info.Event, err error
 
 // Streams all events that occur that satisfy the request into the channel
 // that is passed
-func (self *Client) EventStreamingInfo(name string, einfo chan *info.Event) (err error) {
+func (self *Client) EventStreamingInfo(name string, einfo chan *v1.Event) (err error) {
 	u := self.eventsInfoUrl(name)
 	if err = self.getEventStreamingData(u, einfo); err != nil {
 		return
@@ -70,9 +70,9 @@ func (self *Client) EventStreamingInfo(name string, einfo chan *info.Event) (err
 // MachineInfo returns the JSON machine information for this client.
 // A non-nil error result indicates a problem with obtaining
 // the JSON machine information data.
-func (self *Client) MachineInfo() (minfo *info.MachineInfo, err error) {
+func (self *Client) MachineInfo() (minfo *v1.MachineInfo, err error) {
 	u := self.machineInfoUrl()
-	ret := new(info.MachineInfo)
+	ret := new(v1.MachineInfo)
 	if err = self.httpGetJsonData(ret, nil, u, "machine info"); err != nil {
 		return
 	}
@@ -82,9 +82,9 @@ func (self *Client) MachineInfo() (minfo *info.MachineInfo, err error) {
 
 // ContainerInfo returns the JSON container information for the specified
 // container and request.
-func (self *Client) ContainerInfo(name string, query *info.ContainerInfoRequest) (cinfo *info.ContainerInfo, err error) {
+func (self *Client) ContainerInfo(name string, query *v1.ContainerInfoRequest) (cinfo *v1.ContainerInfo, err error) {
 	u := self.containerInfoUrl(name)
-	ret := new(info.ContainerInfo)
+	ret := new(v1.ContainerInfo)
 	if err = self.httpGetJsonData(ret, query, u, fmt.Sprintf("container info for %q", name)); err != nil {
 		return
 	}
@@ -93,12 +93,12 @@ func (self *Client) ContainerInfo(name string, query *info.ContainerInfoRequest)
 }
 
 // Returns the information about all subcontainers (recursive) of the specified container (including itself).
-func (self *Client) SubcontainersInfo(name string, query *info.ContainerInfoRequest) ([]info.ContainerInfo, error) {
+func (self *Client) SubcontainersInfo(name string, query *v1.ContainerInfoRequest) ([]v1.ContainerInfo, error) {
-	var response []info.ContainerInfo
+	var response []v1.ContainerInfo
 	url := self.subcontainersInfoUrl(name)
 	err := self.httpGetJsonData(&response, query, url, fmt.Sprintf("subcontainers container info for %q", name))
 	if err != nil {
-		return []info.ContainerInfo{}, err
+		return []v1.ContainerInfo{}, err
 
 	}
 	return response, nil
@@ -106,9 +106,9 @@ func (self *Client) SubcontainersInfo(name string, query *info.ContainerInfoRequ
 
 // Returns the JSON container information for the specified
 // Docker container and request.
-func (self *Client) DockerContainer(name string, query *info.ContainerInfoRequest) (cinfo info.ContainerInfo, err error) {
+func (self *Client) DockerContainer(name string, query *v1.ContainerInfoRequest) (cinfo v1.ContainerInfo, err error) {
 	u := self.dockerInfoUrl(name)
-	ret := make(map[string]info.ContainerInfo)
+	ret := make(map[string]v1.ContainerInfo)
 	if err = self.httpGetJsonData(&ret, query, u, fmt.Sprintf("Docker container info for %q", name)); err != nil {
 		return
 	}
@@ -123,13 +123,13 @@ func (self *Client) DockerContainer(name string, query *info.ContainerInfoReques
 }
 
 // Returns the JSON container information for all Docker containers.
-func (self *Client) AllDockerContainers(query *info.ContainerInfoRequest) (cinfo []info.ContainerInfo, err error) {
+func (self *Client) AllDockerContainers(query *v1.ContainerInfoRequest) (cinfo []v1.ContainerInfo, err error) {
 	u := self.dockerInfoUrl("/")
-	ret := make(map[string]info.ContainerInfo)
+	ret := make(map[string]v1.ContainerInfo)
 	if err = self.httpGetJsonData(&ret, query, u, "all Docker containers info"); err != nil {
 		return
 	}
-	cinfo = make([]info.ContainerInfo, 0, len(ret))
+	cinfo = make([]v1.ContainerInfo, 0, len(ret))
 	for _, cont := range ret {
 		cinfo = append(cinfo, cont)
 	}
@@ -191,7 +191,7 @@ func (self *Client) httpGetJsonData(data, postData interface{}, url, infoName st
 	return nil
 }
 
-func (self *Client) getEventStreamingData(url string, einfo chan *info.Event) error {
+func (self *Client) getEventStreamingData(url string, einfo chan *v1.Event) error {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return err
@@ -205,7 +205,7 @@ func (self *Client) getEventStreamingData(url string, einfo chan *info.Event) er
 	}
 
 	dec := json.NewDecoder(resp.Body)
-	var m *info.Event = &info.Event{}
+	var m *v1.Event = &v1.Event{}
 	for {
 		err := dec.Decode(m)
 		if err != nil {

client/v2/client.go

@@ -25,7 +25,7 @@ import (
 	"strings"
 
 	v1 "github.com/google/cadvisor/info/v1"
-	info "github.com/google/cadvisor/info/v2"
+	"github.com/google/cadvisor/info/v2"
 )
 
 // Client represents the base URL for a cAdvisor client.
@@ -40,7 +40,7 @@ func NewClient(url string) (*Client, error) {
 	}
 
 	return &Client{
-		baseUrl: fmt.Sprintf("%sapi/v2.0/", url),
+		baseUrl: fmt.Sprintf("%sapi/v2.1/", url),
 	}, nil
 }
 
@@ -57,6 +57,16 @@ func (self *Client) MachineInfo() (minfo *v1.MachineInfo, err error) {
 	return
 }
 
+// MachineStats returns the JSON machine statistics for this client.
+// A non-nil error result indicates a problem with obtaining
+// the JSON machine information data.
+func (self *Client) MachineStats() ([]v2.MachineStats, error) {
+	var ret []v2.MachineStats
+	u := self.machineStatsUrl()
+	err := self.httpGetJsonData(&ret, nil, u, "machine stats")
+	return ret, err
+}
+
 // VersionInfo returns the version info for cAdvisor.
 func (self *Client) VersionInfo() (version string, err error) {
 	u := self.versionInfoUrl()
@@ -65,9 +75,9 @@ func (self *Client) VersionInfo() (version string, err error) {
 }
 
 // Attributes returns hardware and software attributes of the machine.
-func (self *Client) Attributes() (attr *info.Attributes, err error) {
+func (self *Client) Attributes() (attr *v2.Attributes, err error) {
 	u := self.attributesUrl()
-	ret := new(info.Attributes)
+	ret := new(v2.Attributes)
 	if err = self.httpGetJsonData(ret, nil, u, "attributes"); err != nil {
 		return
 	}
@@ -79,6 +89,10 @@ func (self *Client) machineInfoUrl() string {
 	return self.baseUrl + path.Join("machine")
 }
 
+func (self *Client) machineStatsUrl() string {
+	return self.baseUrl + path.Join("machinestats")
+}
+
 func (self *Client) versionInfoUrl() string {
 	return self.baseUrl + path.Join("version")
 }

client/v2/client_test.go

@@ -22,9 +22,11 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
-	info "github.com/google/cadvisor/info/v1"
+	"github.com/google/cadvisor/info/v1"
-	infoV2 "github.com/google/cadvisor/info/v2"
+	"github.com/google/cadvisor/info/v2"
+	"github.com/stretchr/testify/assert"
 
 	"github.com/kr/pretty"
 )
@@ -43,11 +45,11 @@ func testGetJsonData(
 	return nil
 }
 
-func cadvisorTestClient(path string, expectedPostObj *info.ContainerInfoRequest, replyObj interface{}, t *testing.T) (*Client, *httptest.Server, error) {
+func cadvisorTestClient(path string, expectedPostObj *v1.ContainerInfoRequest, replyObj interface{}, t *testing.T) (*Client, *httptest.Server, error) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path == path {
 			if expectedPostObj != nil {
-				expectedPostObjEmpty := new(info.ContainerInfoRequest)
+				expectedPostObjEmpty := new(v1.ContainerInfoRequest)
 				decoder := json.NewDecoder(r.Body)
 				if err := decoder.Decode(expectedPostObjEmpty); err != nil {
 					t.Errorf("Received invalid object: %v", err)
@@ -60,7 +62,7 @@ func cadvisorTestClient(path string, expectedPostObj *info.ContainerInfoRequest,
 			}
 			encoder := json.NewEncoder(w)
 			encoder.Encode(replyObj)
-		} else if r.URL.Path == "/api/v2.0/version" {
+		} else if r.URL.Path == "/api/v2.1/version" {
 			fmt.Fprintf(w, "0.1.2")
 		} else {
 			w.WriteHeader(http.StatusNotFound)
@@ -77,11 +79,11 @@ func cadvisorTestClient(path string, expectedPostObj *info.ContainerInfoRequest,
 
 // TestGetMachineInfo performs one test to check if MachineInfo()
 // in a cAdvisor client returns the correct result.
-func TestGetMachineinfo(t *testing.T) {
+func TestGetMachineInfo(t *testing.T) {
-	minfo := &info.MachineInfo{
+	mv1 := &v1.MachineInfo{
 		NumCores:       8,
 		MemoryCapacity: 31625871360,
-		DiskMap: map[string]info.DiskInfo{
+		DiskMap: map[string]v1.DiskInfo{
 			"8:0": {
 				Name:  "sda",
 				Major: 8,
@@ -90,7 +92,7 @@ func TestGetMachineinfo(t *testing.T) {
 			},
 		},
 	}
-	client, server, err := cadvisorTestClient("/api/v2.0/machine", nil, minfo, t)
+	client, server, err := cadvisorTestClient("/api/v2.1/machine", nil, mv1, t)
 	if err != nil {
 		t.Fatalf("unable to get a client %v", err)
 	}
@@ -99,14 +101,14 @@ func TestGetMachineinfo(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !reflect.DeepEqual(returned, minfo) {
+	if !reflect.DeepEqual(returned, mv1) {
-		t.Fatalf("received unexpected machine info")
+		t.Fatalf("received unexpected machine v1")
 	}
 }
 
-// TestGetVersionInfo performs one test to check if VersionInfo()
+// TestGetVersionV1 performs one test to check if VersionV1()
 // in a cAdvisor client returns the correct result.
-func TestGetVersioninfo(t *testing.T) {
+func TestGetVersionv1(t *testing.T) {
 	version := "0.1.2"
 	client, server, err := cadvisorTestClient("", nil, version, t)
 	if err != nil {
@@ -118,21 +120,21 @@ func TestGetVersioninfo(t *testing.T) {
 		t.Fatal(err)
 	}
 	if returned != version {
-		t.Fatalf("received unexpected version info")
+		t.Fatalf("received unexpected version v1")
 	}
 }
 
 // TestAttributes performs one test to check if Attributes()
 // in a cAdvisor client returns the correct result.
 func TestGetAttributes(t *testing.T) {
-	attr := &infoV2.Attributes{
+	attr := &v2.Attributes{
 		KernelVersion:      "3.3.0",
 		ContainerOsVersion: "Ubuntu 14.4",
 		DockerVersion:      "Docker 1.5",
 		CadvisorVersion:    "0.1.2",
 		NumCores:           8,
 		MemoryCapacity:     31625871360,
-		DiskMap: map[string]info.DiskInfo{
+		DiskMap: map[string]v1.DiskInfo{
 			"8:0": {
 				Name:  "sda",
 				Major: 8,
@@ -141,7 +143,7 @@ func TestGetAttributes(t *testing.T) {
 			},
 		},
 	}
-	client, server, err := cadvisorTestClient("/api/v2.0/attributes", nil, attr, t)
+	client, server, err := cadvisorTestClient("/api/v2.1/attributes", nil, attr, t)
 	if err != nil {
 		t.Fatalf("unable to get a client %v", err)
 	}
@@ -155,6 +157,43 @@ func TestGetAttributes(t *testing.T) {
 	}
 }
 
+// TestMachineStats performs one test to check if MachineStats()
+// in a cAdvisor client returns the correct result.
+func TestMachineStats(t *testing.T) {
+	machineStats := []v2.MachineStats{
+		{
+			Timestamp: time.Now(),
+			Cpu: &v1.CpuStats{
+				Usage: v1.CpuUsage{
+					Total: 100000,
+				},
+				LoadAverage: 10,
+			},
+			Filesystem: []v2.MachineFsStats{
+				{
+					Device: "sda1",
+				},
+			},
+		},
+	}
+	client, server, err := cadvisorTestClient("/api/v2.1/machinestats", nil, &machineStats, t)
+	if err != nil {
+		t.Fatalf("unable to get a client %v", err)
+	}
+	defer server.Close()
+	returned, err := client.MachineStats()
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Len(t, returned, len(machineStats))
+	if !reflect.DeepEqual(returned[0].Cpu, machineStats[0].Cpu) {
+		t.Fatalf("received unexpected machine stats\nExp: %+v\nAct: %+v", machineStats, returned)
+	}
+	if !reflect.DeepEqual(returned[0].Filesystem, machineStats[0].Filesystem) {
+		t.Fatalf("received unexpected machine stats\nExp: %+v\nAct: %+v", machineStats, returned)
+	}
+}
+
 func TestRequestFails(t *testing.T) {
 	errorText := "there was an error"
 	// Setup a server that simply fails.

container/docker/fsHandler.go

@@ -26,17 +26,19 @@ import (
 
 type fsHandler interface {
 	start()
-	usage() uint64
+	usage() (uint64, uint64)
 	stop()
 }
 
 type realFsHandler struct {
 	sync.RWMutex
-	lastUpdate  time.Time
+	lastUpdate     time.Time
-	usageBytes  uint64
+	usageBytes     uint64
-	period      time.Duration
+	baseUsageBytes uint64
-	storageDirs []string
+	period         time.Duration
-	fsInfo      fs.FsInfo
+	rootfs         string
+	extraDir       string
+	fsInfo         fs.FsInfo
 	// Tells the container to stop.
 	stopChan chan struct{}
 }
@@ -45,14 +47,16 @@ const longDu = time.Second
 
 var _ fsHandler = &realFsHandler{}
 
-func newFsHandler(period time.Duration, storageDirs []string, fsInfo fs.FsInfo) fsHandler {
+func newFsHandler(period time.Duration, rootfs, extraDir string, fsInfo fs.FsInfo) fsHandler {
 	return &realFsHandler{
-		lastUpdate:  time.Time{},
+		lastUpdate:     time.Time{},
-		usageBytes:  0,
+		usageBytes:     0,
-		period:      period,
+		baseUsageBytes: 0,
-		storageDirs: storageDirs,
+		period:         period,
-		fsInfo:      fsInfo,
+		rootfs:         rootfs,
-		stopChan:    make(chan struct{}, 1),
+		extraDir:       extraDir,
+		fsInfo:         fsInfo,
+		stopChan:       make(chan struct{}, 1),
 	}
 }
 
@@ -61,19 +65,22 @@ func (fh *realFsHandler) needsUpdate() bool {
 }
 
 func (fh *realFsHandler) update() error {
-	var usage uint64
+	// TODO(vishh): Add support for external mounts.
-	for _, dir := range fh.storageDirs {
+	baseUsage, err := fh.fsInfo.GetDirUsage(fh.rootfs)
-		// TODO(Vishh): Add support for external mounts.
+	if err != nil {
-		dirUsage, err := fh.fsInfo.GetDirUsage(dir)
+		return err
-		if err != nil {
-			return err
-		}
-		usage += dirUsage
 	}
+
+	extraDirUsage, err := fh.fsInfo.GetDirUsage(fh.extraDir)
+	if err != nil {
+		return err
+	}
+
 	fh.Lock()
 	defer fh.Unlock()
 	fh.lastUpdate = time.Now()
-	fh.usageBytes = usage
+	fh.usageBytes = baseUsage + extraDirUsage
+	fh.baseUsageBytes = baseUsage
 	return nil
 }
 
@@ -90,7 +97,7 @@ func (fh *realFsHandler) trackUsage() {
 			}
 			duration := time.Since(start)
 			if duration > longDu {
-				glog.V(3).Infof("`du` on following dirs took %v: %v", duration, fh.storageDirs)
+				glog.V(3).Infof("`du` on following dirs took %v: %v", duration, []string{fh.rootfs, fh.extraDir})
 			}
 		}
 	}
@@ -104,8 +111,8 @@ func (fh *realFsHandler) stop() {
 	close(fh.stopChan)
 }
 
-func (fh *realFsHandler) usage() uint64 {
+func (fh *realFsHandler) usage() (baseUsageBytes, totalUsageBytes uint64) {
 	fh.RLock()
 	defer fh.RUnlock()
-	return fh.usageBytes
+	return fh.baseUsageBytes, fh.usageBytes
 }

container/docker/handler.go

@@ -59,9 +59,9 @@ type dockerContainerHandler struct {
 	// Manager of this container's cgroups.
 	cgroupManager cgroups.Manager
 
-	storageDriver storageDriver
+	storageDriver    storageDriver
-	fsInfo        fs.FsInfo
+	fsInfo           fs.FsInfo
-	storageDirs   []string
+	rootfsStorageDir string
 
 	// Time at which this container was created.
 	creationTime time.Time
@@ -118,14 +118,13 @@ func newDockerContainerHandler(
 	id := ContainerNameToDockerId(name)
 
 	// Add the Containers dir where the log files are stored.
-	storageDirs := []string{path.Join(*dockerRootDir, pathToContainersDir, id)}
+	otherStorageDir := path.Join(*dockerRootDir, pathToContainersDir, id)
-
+	var rootfsStorageDir string
 	switch storageDriver {
 	case aufsStorageDriver:
-		// Add writable layer for aufs.
+		rootfsStorageDir = path.Join(*dockerRootDir, pathToAufsDir, id)
-		storageDirs = append(storageDirs, path.Join(*dockerRootDir, pathToAufsDir, id))
 	case overlayStorageDriver:
-		storageDirs = append(storageDirs, path.Join(*dockerRootDir, pathToOverlayDir, id))
+		rootfsStorageDir = path.Join(*dockerRootDir, pathToOverlayDir, id)
 	}
 
 	handler := &dockerContainerHandler{
@@ -138,8 +137,8 @@ func newDockerContainerHandler(
 		storageDriver:      storageDriver,
 		fsInfo:             fsInfo,
 		rootFs:             rootFs,
-		storageDirs:        storageDirs,
+		rootfsStorageDir:   rootfsStorageDir,
-		fsHandler:          newFsHandler(time.Minute, storageDirs, fsInfo),
+		fsHandler:          newFsHandler(time.Minute, rootfsStorageDir, otherStorageDir, fsInfo),
 	}
 
 	// We assume that if Inspect fails then the container is not known to docker.
@@ -274,9 +273,7 @@ func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error
 		return nil
 	}
 
-	// As of now we assume that all the storage dirs are on the same device.
+	deviceInfo, err := self.fsInfo.GetDirFsDevice(self.rootfsStorageDir)
-	// The first storage dir will be that of the image layers.
-	deviceInfo, err := self.fsInfo.GetDirFsDevice(self.storageDirs[0])
 	if err != nil {
 		return err
 	}
@@ -296,7 +293,7 @@ func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error
 
 	fsStat := info.FsStats{Device: deviceInfo.Device, Limit: limit}
 
-	fsStat.Usage = self.fsHandler.usage()
+	fsStat.BaseUsage, fsStat.Usage = self.fsHandler.usage()
 	stats.Filesystem = append(stats.Filesystem, fsStat)
 
 	return nil

info/v1/container.go

@@ -398,6 +398,10 @@ type FsStats struct {
 	// Number of bytes that is consumed by the container on this filesystem.
 	Usage uint64 `json:"usage"`
 
+	// Base Usage that is consumed by the container's writable layer.
+	// This field is only applicable for docker container's as of now.
+	BaseUsage uint64 `json:"base_usage"`
+
 	// Number of bytes available for non-root user.
 	Available uint64 `json:"available"`
 

info/v2/container.go

@@ -95,7 +95,7 @@ type ContainerSpec struct {
 	Image string `json:"image,omitempty"`
 }
 
-type ContainerStats struct {
+type DeprecatedContainerStats struct {
 	// The time of this stat point.
 	Timestamp time.Time `json:"timestamp"`
 	// CPU statistics
@@ -124,6 +124,28 @@ type ContainerStats struct {
 	CustomMetrics    map[string][]v1.MetricVal `json:"custom_metrics,omitempty"`
 }
 
+type ContainerStats struct {
+	// The time of this stat point.
+	Timestamp time.Time `json:"timestamp"`
+	// CPU statistics
+	// In nanoseconds (aggregated)
+	Cpu *v1.CpuStats `json:"cpu,omitempty"`
+	// In nanocores per second (instantaneous)
+	CpuInst *CpuInstStats `json:"cpu_inst,omitempty"`
+	// Disk IO statistics
+	DiskIo *v1.DiskIoStats `json:"diskio,omitempty"`
+	// Memory statistics
+	Memory *v1.MemoryStats `json:"memory,omitempty"`
+	// Network statistics
+	Network *NetworkStats `json:"network,omitempty"`
+	// Filesystem statistics
+	Filesystem *FilesystemStats `json:"filesystem,omitempty"`
+	// Task load statistics
+	Load *v1.LoadStats `json:"load_stats,omitempty"`
+	// Custom Metrics
+	CustomMetrics map[string][]v1.MetricVal `json:"custom_metrics,omitempty"`
+}
+
 type Percentiles struct {
 	// Indicates whether the stats are present or not.
 	// If true, values below do not have any data.
@@ -262,3 +284,11 @@ type CpuInstUsage struct {
 	// Unit: nanocores per second
 	System uint64 `json:"system"`
 }
+
+// Filesystem usage statistics.
+type FilesystemStats struct {
+	// Total Number of bytes consumed by container.
+	TotalUsageBytes *uint64 `json:"totalUsageBytes,omitempty"`
+	// Number of bytes consumed by a container through its root filesystem.
+	BaseUsageBytes *uint64 `json:"baseUsageBytes,omitempty"`
+}

info/v2/conversion.go

@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
+// Copyright 2015 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Utilities for converting v1 structs to v2 structs.
 package v2
 
 import (
@@ -23,6 +22,217 @@ import (
 	"github.com/google/cadvisor/info/v1"
 )
 
+func machineFsStatsFromV1(fsStats []v1.FsStats) []MachineFsStats {
+	var result []MachineFsStats
+	for _, stat := range fsStats {
+		readDuration := time.Millisecond * time.Duration(stat.ReadTime)
+		writeDuration := time.Millisecond * time.Duration(stat.WriteTime)
+		ioDuration := time.Millisecond * time.Duration(stat.IoTime)
+		weightedDuration := time.Millisecond * time.Duration(stat.WeightedIoTime)
+		result = append(result, MachineFsStats{
+			Device:    stat.Device,
+			Capacity:  &stat.Limit,
+			Usage:     &stat.Usage,
+			Available: &stat.Available,
+			DiskStats: DiskStats{
+				ReadsCompleted:     &stat.ReadsCompleted,
+				ReadsMerged:        &stat.ReadsMerged,
+				SectorsRead:        &stat.SectorsRead,
+				ReadDuration:       &readDuration,
+				WritesCompleted:    &stat.WritesCompleted,
+				WritesMerged:       &stat.WritesMerged,
+				SectorsWritten:     &stat.SectorsWritten,
+				WriteDuration:      &writeDuration,
+				IoInProgress:       &stat.IoInProgress,
+				IoDuration:         &ioDuration,
+				WeightedIoDuration: &weightedDuration,
+			},
+		})
+	}
+	return result
+}
+
+func MachineStatsFromV1(cont *v1.ContainerInfo) []MachineStats {
+	var stats []MachineStats
+	var last *v1.ContainerStats
+	for _, val := range cont.Stats {
+		stat := MachineStats{
+			Timestamp: val.Timestamp,
+		}
+		if cont.Spec.HasCpu {
+			stat.Cpu = &val.Cpu
+			cpuInst, err := InstCpuStats(last, val)
+			if err != nil {
+				glog.Warningf("Could not get instant cpu stats: %v", err)
+			} else {
+				stat.CpuInst = cpuInst
+			}
+			last = val
+		}
+		if cont.Spec.HasMemory {
+			stat.Memory = &val.Memory
+		}
+		if cont.Spec.HasNetwork {
+			stat.Network = &NetworkStats{
+				// FIXME: Use reflection instead.
+				Tcp:        TcpStat(val.Network.Tcp),
+				Tcp6:       TcpStat(val.Network.Tcp6),
+				Interfaces: val.Network.Interfaces,
+			}
+		}
+		if cont.Spec.HasFilesystem {
+			stat.Filesystem = machineFsStatsFromV1(val.Filesystem)
+		}
+		// TODO(rjnagal): Handle load stats.
+		stats = append(stats, stat)
+	}
+	return stats
+}
+
+func ContainerStatsFromV1(spec *v1.ContainerSpec, stats []*v1.ContainerStats) []*ContainerStats {
+	newStats := make([]*ContainerStats, 0, len(stats))
+	var last *v1.ContainerStats
+	for _, val := range stats {
+		stat := &ContainerStats{
+			Timestamp: val.Timestamp,
+		}
+		if spec.HasCpu {
+			stat.Cpu = &val.Cpu
+			cpuInst, err := InstCpuStats(last, val)
+			if err != nil {
+				glog.Warningf("Could not get instant cpu stats: %v", err)
+			} else {
+				stat.CpuInst = cpuInst
+			}
+			last = val
+		}
+		if spec.HasMemory {
+			stat.Memory = &val.Memory
+		}
+		if spec.HasNetwork {
+			stat.Network.Interfaces = val.Network.Interfaces
+		}
+		if spec.HasFilesystem {
+			if len(val.Filesystem) == 1 {
+				stat.Filesystem = &FilesystemStats{
+					TotalUsageBytes: &val.Filesystem[0].Usage,
+					BaseUsageBytes:  &val.Filesystem[0].BaseUsage,
+				}
+			} else {
+				// Cannot handle multiple devices per container.
+				glog.Errorf("failed to handle multiple devices for container. Skipping Filesystem stats")
+			}
+		}
+		if spec.HasDiskIo {
+			stat.DiskIo = &val.DiskIo
+		}
+		if spec.HasCustomMetrics {
+			stat.CustomMetrics = val.CustomMetrics
+		}
+		// TODO(rjnagal): Handle load stats.
+		newStats = append(newStats, stat)
+	}
+	return newStats
+}
+
+func DeprecatedStatsFromV1(cont *v1.ContainerInfo) []DeprecatedContainerStats {
+	stats := make([]DeprecatedContainerStats, 0, len(cont.Stats))
+	var last *v1.ContainerStats
+	for _, val := range cont.Stats {
+		stat := DeprecatedContainerStats{
+			Timestamp:        val.Timestamp,
+			HasCpu:           cont.Spec.HasCpu,
+			HasMemory:        cont.Spec.HasMemory,
+			HasNetwork:       cont.Spec.HasNetwork,
+			HasFilesystem:    cont.Spec.HasFilesystem,
+			HasDiskIo:        cont.Spec.HasDiskIo,
+			HasCustomMetrics: cont.Spec.HasCustomMetrics,
+		}
+		if stat.HasCpu {
+			stat.Cpu = val.Cpu
+			cpuInst, err := InstCpuStats(last, val)
+			if err != nil {
+				glog.Warningf("Could not get instant cpu stats: %v", err)
+			} else {
+				stat.CpuInst = cpuInst
+			}
+			last = val
+		}
+		if stat.HasMemory {
+			stat.Memory = val.Memory
+		}
+		if stat.HasNetwork {
+			stat.Network.Interfaces = val.Network.Interfaces
+		}
+		if stat.HasFilesystem {
+			stat.Filesystem = val.Filesystem
+		}
+		if stat.HasDiskIo {
+			stat.DiskIo = val.DiskIo
+		}
+		if stat.HasCustomMetrics {
+			stat.CustomMetrics = val.CustomMetrics
+		}
+		// TODO(rjnagal): Handle load stats.
+		stats = append(stats, stat)
+	}
+	return stats
+}
+
+func InstCpuStats(last, cur *v1.ContainerStats) (*CpuInstStats, error) {
+	if last == nil {
+		return nil, nil
+	}
+	if !cur.Timestamp.After(last.Timestamp) {
+		return nil, fmt.Errorf("container stats move backwards in time")
+	}
+	if len(last.Cpu.Usage.PerCpu) != len(cur.Cpu.Usage.PerCpu) {
+		return nil, fmt.Errorf("different number of cpus")
+	}
+	timeDelta := cur.Timestamp.Sub(last.Timestamp)
+	if timeDelta <= 100*time.Millisecond {
+		return nil, fmt.Errorf("time delta unexpectedly small")
+	}
+	// Nanoseconds to gain precision and avoid having zero seconds if the
+	// difference between the timestamps is just under a second
+	timeDeltaNs := uint64(timeDelta.Nanoseconds())
+	convertToRate := func(lastValue, curValue uint64) (uint64, error) {
+		if curValue < lastValue {
+			return 0, fmt.Errorf("cumulative stats decrease")
+		}
+		valueDelta := curValue - lastValue
+		return (valueDelta * 1e9) / timeDeltaNs, nil
+	}
+	total, err := convertToRate(last.Cpu.Usage.Total, cur.Cpu.Usage.Total)
+	if err != nil {
+		return nil, err
+	}
+	percpu := make([]uint64, len(last.Cpu.Usage.PerCpu))
+	for i := range percpu {
+		var err error
+		percpu[i], err = convertToRate(last.Cpu.Usage.PerCpu[i], cur.Cpu.Usage.PerCpu[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+	user, err := convertToRate(last.Cpu.Usage.User, cur.Cpu.Usage.User)
+	if err != nil {
+		return nil, err
+	}
+	system, err := convertToRate(last.Cpu.Usage.System, cur.Cpu.Usage.System)
+	if err != nil {
+		return nil, err
+	}
+	return &CpuInstStats{
+		Usage: CpuInstUsage{
+			Total:  total,
+			PerCpu: percpu,
+			User:   user,
+			System: system,
+		},
+	}, nil
+}
+
 // Get V2 container spec from v1 container info.
 func ContainerSpecFromV1(specV1 *v1.ContainerSpec, aliases []string, namespace string) ContainerSpec {
 	specV2 := ContainerSpec{
@@ -54,50 +264,6 @@ func ContainerSpecFromV1(specV1 *v1.ContainerSpec, aliases []string, namespace s
 	return specV2
 }
 
-func ContainerStatsFromV1(statsV1 []*v1.ContainerStats, specV1 *v1.ContainerSpec) []*ContainerStats {
-	stats := make([]*ContainerStats, 0, len(statsV1))
-	var last *v1.ContainerStats
-	for _, val := range statsV1 {
-		stat := ContainerStats{
-			Timestamp:        val.Timestamp,
-			HasCpu:           specV1.HasCpu,
-			HasMemory:        specV1.HasMemory,
-			HasNetwork:       specV1.HasNetwork,
-			HasFilesystem:    specV1.HasFilesystem,
-			HasDiskIo:        specV1.HasDiskIo,
-			HasCustomMetrics: specV1.HasCustomMetrics,
-		}
-		if stat.HasCpu {
-			stat.Cpu = val.Cpu
-			cpuInst, err := instCpuStats(last, val)
-			if err != nil {
-				glog.Warningf("Could not get instant cpu stats: %v", err)
-			} else {
-				stat.CpuInst = cpuInst
-			}
-			last = val
-		}
-		if stat.HasMemory {
-			stat.Memory = val.Memory
-		}
-		if stat.HasNetwork {
-			stat.Network.Interfaces = val.Network.Interfaces
-		}
-		if stat.HasFilesystem {
-			stat.Filesystem = val.Filesystem
-		}
-		if stat.HasDiskIo {
-			stat.DiskIo = val.DiskIo
-		}
-		if stat.HasCustomMetrics {
-			stat.CustomMetrics = val.CustomMetrics
-		}
-		// TODO(rjnagal): Handle load stats.
-		stats = append(stats, &stat)
-	}
-	return stats
-}
-
 func instCpuStats(last, cur *v1.ContainerStats) (*CpuInstStats, error) {
 	if last == nil {
 		return nil, nil

info/v2/machine.go

@@ -16,6 +16,8 @@ package v2
 
 import (
 	// TODO(rjnagal): Move structs from v1.
+	"time"
+
 	"github.com/google/cadvisor/info/v1"
 )
 
@@ -86,3 +88,97 @@ func GetAttributes(mi *v1.MachineInfo, vi *v1.VersionInfo) Attributes {
 		InstanceType:       mi.InstanceType,
 	}
 }
+
+// MachineStats contains usage statistics for the entire machine.
+type MachineStats struct {
+	// The time of this stat point.
+	Timestamp time.Time `json:"timestamp"`
+	// In nanoseconds (aggregated)
+	Cpu *v1.CpuStats `json:"cpu,omitempty"`
+	// In nanocores per second (instantaneous)
+	CpuInst *CpuInstStats `json:"cpu_inst,omitempty"`
+	// Memory statistics
+	Memory *v1.MemoryStats `json:"memory,omitempty"`
+	// Network statistics
+	Network *NetworkStats `json:"network,omitempty"`
+	// Filesystem statistics
+	Filesystem []MachineFsStats `json:"filesystem,omitempty"`
+	// Task load statistics
+	Load *v1.LoadStats `json:"load_stats,omitempty"`
+}
+
+// MachineFsStats contains per filesystem capacity and usage information.
+type MachineFsStats struct {
+	// The block device name associated with the filesystem.
+	Device string `json:"device"`
+
+	// Number of bytes that can be consumed on this filesystem.
+	Capacity *uint64 `json:"capacity,omitempty"`
+
+	// Number of bytes that is currently consumed on this filesystem.
+	Usage *uint64 `json:"usage,omitempty"`
+
+	// Number of bytes available for non-root user on this filesystem.
+	Available *uint64 `json:"available,omitempty"`
+
+	// DiskStats for this device.
+	DiskStats `json:"inline"`
+}
+
+// DiskStats contains per partition usage information.
+// This information is only available at the machine level.
+type DiskStats struct {
+	// Number of reads completed
+	// This is the total number of reads completed successfully.
+	ReadsCompleted *uint64 `json:"reads_completed,omitempty"`
+
+	// Number of reads merged
+	// Reads and writes which are adjacent to each other may be merged for
+	// efficiency.  Thus two 4K reads may become one 8K read before it is
+	// ultimately handed to the disk, and so it will be counted (and queued)
+	// as only one I/O.  This field lets you know how often this was done.
+	ReadsMerged *uint64 `json:"reads_merged,omitempty"`
+
+	// Number of sectors read
+	// This is the total number of sectors read successfully.
+	SectorsRead *uint64 `json:"sectors_read,omitempty"`
+
+	// Time spent reading
+	// This is the total number of milliseconds spent by all reads (as
+	// measured from __make_request() to end_that_request_last()).
+	ReadDuration *time.Duration `json:"read_duration,omitempty"`
+
+	// Number of writes completed
+	// This is the total number of writes completed successfully.
+	WritesCompleted *uint64 `json:"writes_completed,omitempty"`
+
+	// Number of writes merged
+	// See the description of reads merged.
+	WritesMerged *uint64 `json:"writes_merged,omitempty"`
+
+	// Number of sectors written
+	// This is the total number of sectors written successfully.
+	SectorsWritten *uint64 `json:"sectors_written,omitempty"`
+
+	// Time spent writing
+	// This is the total number of milliseconds spent by all writes (as
+	// measured from __make_request() to end_that_request_last()).
+	WriteDuration *time.Duration `json:"write_duration,omitempty"`
+
+	// Number of I/Os currently in progress
+	// The only field that should go to zero. Incremented as requests are
+	// given to appropriate struct request_queue and decremented as they finish.
+	IoInProgress *uint64 `json:"io_in_progress,omitempty"`
+
+	// Time spent doing I/Os
+	// This field increases so long as field 9 is nonzero.
+	IoDuration *time.Duration `json:"io_duration,omitempty"`
+
+	// weighted time spent doing I/Os
+	// This field is incremented at each I/O start, I/O completion, I/O
+	// merge, or read of these stats by the number of I/Os in progress
+	// (field 9) times the number of milliseconds spent doing I/O since the
+	// last update of this field.  This can provide an easy measure of both
+	// I/O completion time and the backlog that may be accumulating.
+	WeightedIoDuration *time.Duration `json:"weighted_io_duration,omitempty"`
+}

integration/tests/api/machinestats_test.go

@@ -0,0 +1,54 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"testing"
+	"time"
+
+	"github.com/google/cadvisor/integration/framework"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMachineStatsIsReturned(t *testing.T) {
+	fm := framework.New(t)
+	defer fm.Cleanup()
+
+	machineStats, err := fm.Cadvisor().ClientV2().MachineStats()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	as := assert.New(t)
+	for _, stat := range machineStats {
+		as.NotEqual(stat.Timestamp, time.Time{})
+		as.True(stat.Cpu.Usage.Total > 0)
+		as.True(len(stat.Cpu.Usage.PerCpu) > 0)
+		if stat.CpuInst != nil {
+			as.True(stat.CpuInst.Usage.Total > 0)
+		}
+		as.True(stat.Memory.Usage > 0)
+		for _, nStat := range stat.Network.Interfaces {
+			as.NotEqual(nStat.Name, "")
+			as.NotEqual(nStat.RxBytes, 0)
+		}
+		for _, fsStat := range stat.Filesystem {
+			as.NotEqual(fsStat.Device, "")
+			as.NotNil(fsStat.Capacity)
+			as.NotNil(fsStat.Usage)
+			as.NotNil(fsStat.ReadsCompleted)
+		}
+	}
+}

manager/manager.go

@@ -424,7 +424,7 @@ func (self *manager) GetContainerInfoV2(containerName string, options v2.Request
 
 		infos[name] = v2.ContainerInfo{
 			Spec:  self.getV2Spec(cinfo),
-			Stats: v2.ContainerStatsFromV1(stats, &cinfo.Spec),
+			Stats: v2.ContainerStatsFromV1(&cinfo.Spec, stats),
 		}
 	}