Merge pull request #427 from rjnagal/master

Add percentiles utility methods.

utils/percentiles.go

@@ -0,0 +1,135 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"math"
+	"sort"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/google/cadvisor/info"
+)
+
+const milliSecondsToNanoSeconds = 1000000
+const secondsToMilliSeconds = 1000
+
+type uint64Slice []uint64
+
+func (a uint64Slice) Len() int           { return len(a) }
+func (a uint64Slice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a uint64Slice) Less(i, j int) bool { return a[i] < a[j] }
+
+// TODO(rjnagal): Move out when we update API.
+type Percentiles struct {
+	// Average over the collected sample.
+	Mean uint64 `json:"mean"`
+	// Max seen over the collected sample.
+	Max uint64 `json:"max"`
+	// 90th percentile over the collected sample.
+	Ninety uint64 `json:"ninety"`
+}
+
+// Get 90th percentile of the provided samples. Round to integer.
+func (self uint64Slice) Get90Percentile() uint64 {
+	count := self.Len()
+	if count == 0 {
+		return 0
+	}
+	sort.Sort(self)
+	n := float64(0.9 * (float64(count) + 1))
+	idx, frac := math.Modf(n)
+	index := int(idx)
+	percentile := float64(self[index-1])
+	if index > 1 || index < count {
+		percentile += frac * float64(self[index]-self[index-1])
+	}
+	return uint64(percentile)
+}
+
+type Mean struct {
+	// current count.
+	count uint64
+	// current mean.
+	Mean float64
+}
+
+func (self *Mean) Add(value uint64) {
+	self.count++
+	if self.count == 1 {
+		self.Mean = float64(value)
+		return
+	}
+	c := float64(self.count)
+	v := float64(value)
+	self.Mean = (self.Mean*(c-1) + v) / c
+}
+
+// Returns cpu and memory usage percentiles.
+func GetPercentiles(stats []*info.ContainerStats) (Percentiles, Percentiles) {
+	lastCpu := uint64(0)
+	lastTime := time.Time{}
+	memorySamples := make(uint64Slice, 0, len(stats))
+	cpuSamples := make(uint64Slice, 0, len(stats)-1)
+	numSamples := 0
+	memoryMean := Mean{count: 0, Mean: 0}
+	cpuMean := Mean{count: 0, Mean: 0}
+	memoryPercentiles := Percentiles{}
+	cpuPercentiles := Percentiles{}
+	for _, stat := range stats {
+		var elapsed int64
+		time := stat.Timestamp
+		if !lastTime.IsZero() {
+			elapsed = time.UnixNano() - lastTime.UnixNano()
+			if elapsed < 10*milliSecondsToNanoSeconds {
+				glog.Infof("Elapsed time too small: %d ns: time now %s last %s", elapsed, time.String(), lastTime.String())
+				continue
+			}
+		}
+		numSamples++
+		cpuNs := stat.Cpu.Usage.Total
+		// Ignore actual usage and only focus on working set.
+		memory := stat.Memory.WorkingSet
+		if memory > memoryPercentiles.Max {
+			memoryPercentiles.Max = memory
+		}
+		glog.V(2).Infof("Read sample: cpu %d, memory %d", cpuNs, memory)
+		memoryMean.Add(memory)
+		memorySamples = append(memorySamples, memory)
+		if lastTime.IsZero() {
+			lastCpu = cpuNs
+			lastTime = time
+			continue
+		}
+		cpuRate := (cpuNs - lastCpu) * secondsToMilliSeconds / uint64(elapsed)
+		if cpuRate < 0 {
+			glog.Infof("cpu rate too small: %f ns", cpuRate)
+			continue
+		}
+		glog.V(2).Infof("Adding cpu rate sample : %d", cpuRate)
+		lastCpu = cpuNs
+		lastTime = time
+		cpuSamples = append(cpuSamples, cpuRate)
+		if cpuRate > cpuPercentiles.Max {
+			cpuPercentiles.Max = cpuRate
+		}
+		cpuMean.Add(cpuRate)
+	}
+	cpuPercentiles.Mean = uint64(cpuMean.Mean)
+	memoryPercentiles.Mean = uint64(memoryMean.Mean)
+	cpuPercentiles.Ninety = cpuSamples.Get90Percentile()
+	memoryPercentiles.Ninety = memorySamples.Get90Percentile()
+	return cpuPercentiles, memoryPercentiles
+}

utils/percentiles_test.go

@@ -0,0 +1,144 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"testing"
+	"time"
+
+	"github.com/google/cadvisor/info"
+)
+
+const Nanosecond = 1000000000
+
+func Test90Percentile(t *testing.T) {
+	N := 100
+	stats := make(uint64Slice, 0, N)
+	for i := N; i > 0; i-- {
+		stats = append(stats, uint64(i))
+	}
+	p := stats.Get90Percentile()
+	if p != 90 {
+		t.Errorf("90th percentile is %d, should be 90.", p)
+	}
+	// 90p should be between 94 and 95. Promoted to 95.
+	N = 105
+	for i := 101; i <= N; i++ {
+		stats = append(stats, uint64(i))
+	}
+	p = stats.Get90Percentile()
+	if p != 95 {
+		t.Errorf("90th percentile is %d, should be 95.", p)
+	}
+}
+
+func TestMean(t *testing.T) {
+	var i, N uint64
+	N = 100
+	mean := Mean{count: 0, Mean: 0}
+	for i = 1; i < N; i++ {
+		mean.Add(i)
+	}
+	if mean.Mean != 50.0 {
+		t.Errorf("Mean is %f, should be 50.0", mean.Mean)
+	}
+}
+
+func TestAggregates(t *testing.T) {
+	N := uint64(100)
+	var i uint64
+	ct := time.Now()
+	stats := make([]*info.ContainerStats, 0, N)
+	for i = 1; i < N; i++ {
+		s := &info.ContainerStats{
+			Cpu:       info.CpuStats{},
+			Timestamp: ct.Add(time.Duration(i) * time.Second),
+			Memory: info.MemoryStats{
+				// Memory grows by a KB every second.
+				WorkingSet: i * 1024,
+			},
+		}
+		// cpu rate is 1 s/s
+		s.Cpu.Usage.Total = i * Nanosecond
+		stats = append(stats, s)
+	}
+	cpu, mem := GetPercentiles(stats)
+	// Cpu mean, max, and 90p should all be 1000 ms/s.
+	cpuExpected := Percentiles{
+		Mean:   1000,
+		Max:    1000,
+		Ninety: 1000,
+	}
+	if cpu != cpuExpected {
+		t.Errorf("cpu stats are %+v. Expected %+v", cpu, cpuExpected)
+	}
+	memExpected := Percentiles{
+		Mean:   50 * 1024,
+		Max:    99 * 1024,
+		Ninety: 90 * 1024,
+	}
+	if mem != memExpected {
+		t.Errorf("memory stats are mean %+v. Expected %+v", mem, memExpected)
+	}
+}
+func TestSamplesCloseInTimeIgnored(t *testing.T) {
+	N := uint64(100)
+	var i uint64
+	ct := time.Now()
+	stats := make([]*info.ContainerStats, 0, N*2)
+	for i = 1; i < N; i++ {
+		s1 := &info.ContainerStats{
+			Cpu:       info.CpuStats{},
+			Timestamp: ct.Add(time.Duration(i) * time.Second),
+			Memory: info.MemoryStats{
+				// Memory grows by a KB every second.
+				WorkingSet: i * 1024,
+			},
+		}
+		// cpu rate is 1 s/s
+		s1.Cpu.Usage.Total = i * Nanosecond
+		stats = append(stats, s1)
+
+		// Add another dummy sample too close in time to the last one.
+		s2 := &info.ContainerStats{
+			Cpu: info.CpuStats{},
+			// Add extra millisecond.
+			Timestamp: ct.Add(time.Duration(i) * time.Second).Add(time.Duration(1) * time.Millisecond),
+			Memory: info.MemoryStats{
+				WorkingSet: i * 1024 * 1024,
+			},
+		}
+		s2.Cpu.Usage.Total = i * 100 * Nanosecond
+		stats = append(stats, s2)
+	}
+	cpu, mem := GetPercentiles(stats)
+	// Cpu mean, max, and 90p should all be 1000 ms/s. All high-value samples are discarded.
+	cpuExpected := Percentiles{
+		Mean:   1000,
+		Max:    1000,
+		Ninety: 1000,
+	}
+	if cpu != cpuExpected {
+		t.Errorf("cpu stats are %+v. Expected %+v", cpu, cpuExpected)
+	}
+	memExpected := Percentiles{
+		Mean:   50 * 1024,
+		Max:    99 * 1024,
+		Ninety: 90 * 1024,
+	}
+	if mem != memExpected {
+		t.Errorf("memory stats are mean %+v. Expected %+v", mem, memExpected)
+	}
+}