diff --git a/api/versions.go b/api/versions.go index ab5d93ee..43839038 100644 --- a/api/versions.go +++ b/api/versions.go @@ -19,6 +19,7 @@ import ( "net/http" "path" "strconv" + "time" "github.com/golang/glog" info "github.com/google/cadvisor/info/v1" @@ -449,8 +450,63 @@ func (self *version2_0) HandleRequest(requestType string, request []string, m ma } } +func instCpuStats(last, cur *info.ContainerStats) (*v2.CpuInstStats, error) { + if last == nil { + return nil, nil + } + if !cur.Timestamp.After(last.Timestamp) { + return nil, fmt.Errorf("container stats move backwards in time") + } + if len(last.Cpu.Usage.PerCpu) != len(cur.Cpu.Usage.PerCpu) { + return nil, fmt.Errorf("different number of cpus") + } + timeDelta := cur.Timestamp.Sub(last.Timestamp) + if timeDelta <= 100*time.Millisecond { + return nil, fmt.Errorf("time delta unexpectedly small") + } + // Nanoseconds to gain precision and avoid having zero seconds if the + // difference between the timestamps is just under a second + timeDeltaNs := uint64(timeDelta.Nanoseconds()) + convertToRate := func(lastValue, curValue uint64) (uint64, error) { + if curValue < lastValue { + return 0, fmt.Errorf("cumulative stats decrease") + } + valueDelta := curValue - lastValue + return (valueDelta * 1e9) / timeDeltaNs, nil + } + total, err := convertToRate(last.Cpu.Usage.Total, cur.Cpu.Usage.Total) + if err != nil { + return nil, err + } + percpu := make([]uint64, len(last.Cpu.Usage.PerCpu)) + for i := range percpu { + var err error + percpu[i], err = convertToRate(last.Cpu.Usage.PerCpu[i], cur.Cpu.Usage.PerCpu[i]) + if err != nil { + return nil, err + } + } + user, err := convertToRate(last.Cpu.Usage.User, cur.Cpu.Usage.User) + if err != nil { + return nil, err + } + system, err := convertToRate(last.Cpu.Usage.System, cur.Cpu.Usage.System) + if err != nil { + return nil, err + } + return &v2.CpuInstStats{ + Usage: v2.CpuInstUsage{ + Total: total, + PerCpu: percpu, + User: user, + System: system, + }, + }, nil +} + func convertStats(cont *info.ContainerInfo) []v2.ContainerStats { - stats := []v2.ContainerStats{} + stats := make([]v2.ContainerStats, 0, len(cont.Stats)) + var last *info.ContainerStats for _, val := range cont.Stats { stat := v2.ContainerStats{ Timestamp: val.Timestamp, @@ -463,6 +519,13 @@ func convertStats(cont *info.ContainerInfo) []v2.ContainerStats { } if stat.HasCpu { stat.Cpu = val.Cpu + cpuInst, err := instCpuStats(last, val) + if err != nil { + glog.Warningf("Could not get instant cpu stats: %v", err) + } else { + stat.CpuInst = cpuInst + } + last = val } if stat.HasMemory { stat.Memory = val.Memory diff --git a/api/versions_test.go b/api/versions_test.go index 0a107858..82675703 100644 --- a/api/versions_test.go +++ b/api/versions_test.go @@ -19,9 +19,11 @@ import ( "net/http" "reflect" "testing" + "time" "github.com/google/cadvisor/events" info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" ) @@ -78,3 +80,170 @@ func TestGetEventRequestDoubleArgument(t *testing.T) { assert.True(t, stream) assert.Nil(t, err) } + +func TestInstCpuStats(t *testing.T) { + tests := []struct { + last *info.ContainerStats + cur *info.ContainerStats + want *v2.CpuInstStats + }{ + // Last is missing + { + nil, + &info.ContainerStats{}, + nil, + }, + // Goes back in time + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(time.Second), + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + }, + nil, + }, + // Zero time delta + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + }, + nil, + }, + // Unexpectedly small time delta + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(30 * time.Millisecond), + }, + nil, + }, + // Different number of cpus + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + PerCpu: []uint64{100, 200}, + }, + }, + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(time.Second), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + PerCpu: []uint64{100, 200, 300}, + }, + }, + }, + nil, + }, + // Stat numbers decrease + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 300, + PerCpu: []uint64{100, 200}, + User: 250, + System: 50, + }, + }, + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(time.Second), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 200, + PerCpu: []uint64{100, 100}, + User: 150, + System: 50, + }, + }, + }, + nil, + }, + // One second elapsed + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 300, + PerCpu: []uint64{100, 200}, + User: 250, + System: 50, + }, + }, + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(time.Second), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 500, + PerCpu: []uint64{200, 300}, + User: 400, + System: 100, + }, + }, + }, + &v2.CpuInstStats{ + Usage: v2.CpuInstUsage{ + Total: 200, + PerCpu: []uint64{100, 100}, + User: 150, + System: 50, + }, + }, + }, + // Two seconds elapsed + { + &info.ContainerStats{ + Timestamp: time.Unix(100, 0), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 300, + PerCpu: []uint64{100, 200}, + User: 250, + System: 50, + }, + }, + }, + &info.ContainerStats{ + Timestamp: time.Unix(100, 0).Add(2 * time.Second), + Cpu: info.CpuStats{ + Usage: info.CpuUsage{ + Total: 500, + PerCpu: []uint64{200, 300}, + User: 400, + System: 100, + }, + }, + }, + &v2.CpuInstStats{ + Usage: v2.CpuInstUsage{ + Total: 100, + PerCpu: []uint64{50, 50}, + User: 75, + System: 25, + }, + }, + }, + } + for _, c := range tests { + got, err := instCpuStats(c.last, c.cur) + if err != nil { + if c.want == nil { + continue + } + t.Errorf("Unexpected error: %v", err) + } + assert.Equal(t, c.want, got) + } +} diff --git a/info/v2/container.go b/info/v2/container.go index 74446890..e82810c8 100644 --- a/info/v2/container.go +++ b/info/v2/container.go @@ -86,8 +86,11 @@ type ContainerStats struct { // The time of this stat point. Timestamp time.Time `json:"timestamp"` // CPU statistics - HasCpu bool `json:"has_cpu"` - Cpu v1.CpuStats `json:"cpu,omitempty"` + HasCpu bool `json:"has_cpu"` + // In nanoseconds (aggregated) + Cpu v1.CpuStats `json:"cpu,omitempty"` + // In nanocores per second (instantaneous) + CpuInst *CpuInstStats `json:"cpu_inst,omitempty"` // Disk IO statistics HasDiskIo bool `json:"has_diskio"` DiskIo v1.DiskIoStats `json:"diskio,omitempty"` @@ -204,3 +207,27 @@ type NetworkStats struct { // Network stats by interface. Interfaces []v1.InterfaceStats `json:"interfaces,omitempty"` } + +// Instantaneous CPU stats +type CpuInstStats struct { + Usage CpuInstUsage `json:"usage"` +} + +// CPU usage time statistics. +type CpuInstUsage struct { + // Total CPU usage. + // Units: nanocores per second + Total uint64 `json:"total"` + + // Per CPU/core usage of the container. + // Unit: nanocores per second + PerCpu []uint64 `json:"per_cpu_usage,omitempty"` + + // Time spent in user space. + // Unit: nanocores per second + User uint64 `json:"user"` + + // Time spent in kernel space. + // Unit: nanocores per second + System uint64 `json:"system"` +}