Add instantaneous cpu metrics to v2

api/versions.go

@@ -19,6 +19,7 @@ import (
 	"net/http"
 	"path"
 	"strconv"
+	"time"
 
 	"github.com/golang/glog"
 	info "github.com/google/cadvisor/info/v1"
@@ -449,8 +450,63 @@ func (self *version2_0) HandleRequest(requestType string, request []string, m ma
 	}
 }
 
+func instCpuStats(last, cur *info.ContainerStats) (*v2.CpuInstStats, error) {
+	if last == nil {
+		return nil, nil
+	}
+	if !cur.Timestamp.After(last.Timestamp) {
+		return nil, fmt.Errorf("container stats move backwards in time")
+	}
+	if len(last.Cpu.Usage.PerCpu) != len(cur.Cpu.Usage.PerCpu) {
+		return nil, fmt.Errorf("different number of cpus")
+	}
+	timeDelta := cur.Timestamp.Sub(last.Timestamp)
+	if timeDelta <= 100*time.Millisecond {
+		return nil, fmt.Errorf("time delta unexpectedly small")
+	}
+	// Nanoseconds to gain precision and avoid having zero seconds if the
+	// difference between the timestamps is just under a second
+	timeDeltaNs := uint64(timeDelta.Nanoseconds())
+	convertToRate := func(lastValue, curValue uint64) (uint64, error) {
+		if curValue < lastValue {
+			return 0, fmt.Errorf("cumulative stats decrease")
+		}
+		valueDelta := curValue - lastValue
+		return (valueDelta * 1e9) / timeDeltaNs, nil
+	}
+	total, err := convertToRate(last.Cpu.Usage.Total, cur.Cpu.Usage.Total)
+	if err != nil {
+		return nil, err
+	}
+	percpu := make([]uint64, len(last.Cpu.Usage.PerCpu))
+	for i := range percpu {
+		var err error
+		percpu[i], err = convertToRate(last.Cpu.Usage.PerCpu[i], cur.Cpu.Usage.PerCpu[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+	user, err := convertToRate(last.Cpu.Usage.User, cur.Cpu.Usage.User)
+	if err != nil {
+		return nil, err
+	}
+	system, err := convertToRate(last.Cpu.Usage.System, cur.Cpu.Usage.System)
+	if err != nil {
+		return nil, err
+	}
+	return &v2.CpuInstStats{
+		Usage: v2.CpuInstUsage{
+			Total:  total,
+			PerCpu: percpu,
+			User:   user,
+			System: system,
+		},
+	}, nil
+}
+
 func convertStats(cont *info.ContainerInfo) []v2.ContainerStats {
-	stats := []v2.ContainerStats{}
+	stats := make([]v2.ContainerStats, 0, len(cont.Stats))
+	var last *info.ContainerStats
 	for _, val := range cont.Stats {
 		stat := v2.ContainerStats{
 			Timestamp:        val.Timestamp,
@@ -463,6 +519,13 @@ func convertStats(cont *info.ContainerInfo) []v2.ContainerStats {
 		}
 		if stat.HasCpu {
 			stat.Cpu = val.Cpu
+			cpuInst, err := instCpuStats(last, val)
+			if err != nil {
+				glog.Warningf("Could not get instant cpu stats: %v", err)
+			} else {
+				stat.CpuInst = cpuInst
+			}
+			last = val
 		}
 		if stat.HasMemory {
 			stat.Memory = val.Memory

api/versions_test.go

@@ -19,9 +19,11 @@ import (
 	"net/http"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/google/cadvisor/events"
 	info "github.com/google/cadvisor/info/v1"
+	"github.com/google/cadvisor/info/v2"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -78,3 +80,170 @@ func TestGetEventRequestDoubleArgument(t *testing.T) {
 	assert.True(t, stream)
 	assert.Nil(t, err)
 }
+
+func TestInstCpuStats(t *testing.T) {
+	tests := []struct {
+		last *info.ContainerStats
+		cur  *info.ContainerStats
+		want *v2.CpuInstStats
+	}{
+		// Last is missing
+		{
+			nil,
+			&info.ContainerStats{},
+			nil,
+		},
+		// Goes back in time
+		{
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0).Add(time.Second),
+			},
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0),
+			},
+			nil,
+		},
+		// Zero time delta
+		{
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0),
+			},
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0),
+			},
+			nil,
+		},
+		// Unexpectedly small time delta
+		{
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0),
+			},
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0).Add(30 * time.Millisecond),
+			},
+			nil,
+		},
+		// Different number of cpus
+		{
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0),
+				Cpu: info.CpuStats{
+					Usage: info.CpuUsage{
+						PerCpu: []uint64{100, 200},
+					},
+				},
+			},
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0).Add(time.Second),
+				Cpu: info.CpuStats{
+					Usage: info.CpuUsage{
+						PerCpu: []uint64{100, 200, 300},
+					},
+				},
+			},
+			nil,
+		},
+		// Stat numbers decrease
+		{
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0),
+				Cpu: info.CpuStats{
+					Usage: info.CpuUsage{
+						Total:  300,
+						PerCpu: []uint64{100, 200},
+						User:   250,
+						System: 50,
+					},
+				},
+			},
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0).Add(time.Second),
+				Cpu: info.CpuStats{
+					Usage: info.CpuUsage{
+						Total:  200,
+						PerCpu: []uint64{100, 100},
+						User:   150,
+						System: 50,
+					},
+				},
+			},
+			nil,
+		},
+		// One second elapsed
+		{
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0),
+				Cpu: info.CpuStats{
+					Usage: info.CpuUsage{
+						Total:  300,
+						PerCpu: []uint64{100, 200},
+						User:   250,
+						System: 50,
+					},
+				},
+			},
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0).Add(time.Second),
+				Cpu: info.CpuStats{
+					Usage: info.CpuUsage{
+						Total:  500,
+						PerCpu: []uint64{200, 300},
+						User:   400,
+						System: 100,
+					},
+				},
+			},
+			&v2.CpuInstStats{
+				Usage: v2.CpuInstUsage{
+					Total:  200,
+					PerCpu: []uint64{100, 100},
+					User:   150,
+					System: 50,
+				},
+			},
+		},
+		// Two seconds elapsed
+		{
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0),
+				Cpu: info.CpuStats{
+					Usage: info.CpuUsage{
+						Total:  300,
+						PerCpu: []uint64{100, 200},
+						User:   250,
+						System: 50,
+					},
+				},
+			},
+			&info.ContainerStats{
+				Timestamp: time.Unix(100, 0).Add(2 * time.Second),
+				Cpu: info.CpuStats{
+					Usage: info.CpuUsage{
+						Total:  500,
+						PerCpu: []uint64{200, 300},
+						User:   400,
+						System: 100,
+					},
+				},
+			},
+			&v2.CpuInstStats{
+				Usage: v2.CpuInstUsage{
+					Total:  100,
+					PerCpu: []uint64{50, 50},
+					User:   75,
+					System: 25,
+				},
+			},
+		},
+	}
+	for _, c := range tests {
+		got, err := instCpuStats(c.last, c.cur)
+		if err != nil {
+			if c.want == nil {
+				continue
+			}
+			t.Errorf("Unexpected error: %v", err)
+		}
+		assert.Equal(t, c.want, got)
+	}
+}

info/v2/container.go

@@ -86,8 +86,11 @@ type ContainerStats struct {
 	// The time of this stat point.
 	Timestamp time.Time `json:"timestamp"`
 	// CPU statistics
-	HasCpu bool        `json:"has_cpu"`
+	HasCpu bool `json:"has_cpu"`
-	Cpu    v1.CpuStats `json:"cpu,omitempty"`
+	// In nanoseconds (aggregated)
+	Cpu v1.CpuStats `json:"cpu,omitempty"`
+	// In nanocores per second (instantaneous)
+	CpuInst *CpuInstStats `json:"cpu_inst,omitempty"`
 	// Disk IO statistics
 	HasDiskIo bool           `json:"has_diskio"`
 	DiskIo    v1.DiskIoStats `json:"diskio,omitempty"`
@@ -204,3 +207,27 @@ type NetworkStats struct {
 	// Network stats by interface.
 	Interfaces []v1.InterfaceStats `json:"interfaces,omitempty"`
 }
+
+// Instantaneous CPU stats
+type CpuInstStats struct {
+	Usage CpuInstUsage `json:"usage"`
+}
+
+// CPU usage time statistics.
+type CpuInstUsage struct {
+	// Total CPU usage.
+	// Units: nanocores per second
+	Total uint64 `json:"total"`
+
+	// Per CPU/core usage of the container.
+	// Unit: nanocores per second
+	PerCpu []uint64 `json:"per_cpu_usage,omitempty"`
+
+	// Time spent in user space.
+	// Unit: nanocores per second
+	User uint64 `json:"user"`
+
+	// Time spent in kernel space.
+	// Unit: nanocores per second
+	System uint64 `json:"system"`
+}