From de0e8e28eb029ede6bf8c2c4a0b15f04e5d6c8b2 Mon Sep 17 00:00:00 2001 From: Rohit Jnagal Date: Thu, 8 Jan 2015 22:56:12 +0000 Subject: [PATCH] Add percentiles utility methods. This is copied verbatim for our older prototype. We'll use it as a starting point for building stats summary. --- utils/percentiles.go | 135 +++++++++++++++++++++++++++++++++++ utils/percentiles_test.go | 144 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 utils/percentiles.go create mode 100644 utils/percentiles_test.go diff --git a/utils/percentiles.go b/utils/percentiles.go new file mode 100644 index 00000000..3f643fe0 --- /dev/null +++ b/utils/percentiles.go @@ -0,0 +1,135 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "math" + "sort" + "time" + + "github.com/golang/glog" + "github.com/google/cadvisor/info" +) + +const milliSecondsToNanoSeconds = 1000000 +const secondsToMilliSeconds = 1000 + +type uint64Slice []uint64 + +func (a uint64Slice) Len() int { return len(a) } +func (a uint64Slice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a uint64Slice) Less(i, j int) bool { return a[i] < a[j] } + +// TODO(rjnagal): Move out when we update API. +type Percentiles struct { + // Average over the collected sample. + Mean uint64 `json:"mean"` + // Max seen over the collected sample. + Max uint64 `json:"max"` + // 90th percentile over the collected sample. + Ninety uint64 `json:"ninety"` +} + +// Get 90th percentile of the provided samples. Round to integer. +func (self uint64Slice) Get90Percentile() uint64 { + count := self.Len() + if count == 0 { + return 0 + } + sort.Sort(self) + n := float64(0.9 * (float64(count) + 1)) + idx, frac := math.Modf(n) + index := int(idx) + percentile := float64(self[index-1]) + if index > 1 || index < count { + percentile += frac * float64(self[index]-self[index-1]) + } + return uint64(percentile) +} + +type Mean struct { + // current count. + count uint64 + // current mean. + Mean float64 +} + +func (self *Mean) Add(value uint64) { + self.count++ + if self.count == 1 { + self.Mean = float64(value) + return + } + c := float64(self.count) + v := float64(value) + self.Mean = (self.Mean*(c-1) + v) / c +} + +// Returns cpu and memory usage percentiles. +func GetPercentiles(stats []*info.ContainerStats) (Percentiles, Percentiles) { + lastCpu := uint64(0) + lastTime := time.Time{} + memorySamples := make(uint64Slice, 0, len(stats)) + cpuSamples := make(uint64Slice, 0, len(stats)-1) + numSamples := 0 + memoryMean := Mean{count: 0, Mean: 0} + cpuMean := Mean{count: 0, Mean: 0} + memoryPercentiles := Percentiles{} + cpuPercentiles := Percentiles{} + for _, stat := range stats { + var elapsed int64 + time := stat.Timestamp + if !lastTime.IsZero() { + elapsed = time.UnixNano() - lastTime.UnixNano() + if elapsed < 10*milliSecondsToNanoSeconds { + glog.Infof("Elapsed time too small: %d ns: time now %s last %s", elapsed, time.String(), lastTime.String()) + continue + } + } + numSamples++ + cpuNs := stat.Cpu.Usage.Total + // Ignore actual usage and only focus on working set. + memory := stat.Memory.WorkingSet + if memory > memoryPercentiles.Max { + memoryPercentiles.Max = memory + } + glog.V(2).Infof("Read sample: cpu %d, memory %d", cpuNs, memory) + memoryMean.Add(memory) + memorySamples = append(memorySamples, memory) + if lastTime.IsZero() { + lastCpu = cpuNs + lastTime = time + continue + } + cpuRate := (cpuNs - lastCpu) * secondsToMilliSeconds / uint64(elapsed) + if cpuRate < 0 { + glog.Infof("cpu rate too small: %f ns", cpuRate) + continue + } + glog.V(2).Infof("Adding cpu rate sample : %d", cpuRate) + lastCpu = cpuNs + lastTime = time + cpuSamples = append(cpuSamples, cpuRate) + if cpuRate > cpuPercentiles.Max { + cpuPercentiles.Max = cpuRate + } + cpuMean.Add(cpuRate) + } + cpuPercentiles.Mean = uint64(cpuMean.Mean) + memoryPercentiles.Mean = uint64(memoryMean.Mean) + cpuPercentiles.Ninety = cpuSamples.Get90Percentile() + memoryPercentiles.Ninety = memorySamples.Get90Percentile() + return cpuPercentiles, memoryPercentiles +} diff --git a/utils/percentiles_test.go b/utils/percentiles_test.go new file mode 100644 index 00000000..2b86278f --- /dev/null +++ b/utils/percentiles_test.go @@ -0,0 +1,144 @@ +// Copyright 2015 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "testing" + "time" + + "github.com/google/cadvisor/info" +) + +const Nanosecond = 1000000000 + +func Test90Percentile(t *testing.T) { + N := 100 + stats := make(uint64Slice, 0, N) + for i := N; i > 0; i-- { + stats = append(stats, uint64(i)) + } + p := stats.Get90Percentile() + if p != 90 { + t.Errorf("90th percentile is %d, should be 90.", p) + } + // 90p should be between 94 and 95. Promoted to 95. + N = 105 + for i := 101; i <= N; i++ { + stats = append(stats, uint64(i)) + } + p = stats.Get90Percentile() + if p != 95 { + t.Errorf("90th percentile is %d, should be 95.", p) + } +} + +func TestMean(t *testing.T) { + var i, N uint64 + N = 100 + mean := Mean{count: 0, Mean: 0} + for i = 1; i < N; i++ { + mean.Add(i) + } + if mean.Mean != 50.0 { + t.Errorf("Mean is %f, should be 50.0", mean.Mean) + } +} + +func TestAggregates(t *testing.T) { + N := uint64(100) + var i uint64 + ct := time.Now() + stats := make([]*info.ContainerStats, 0, N) + for i = 1; i < N; i++ { + s := &info.ContainerStats{ + Cpu: info.CpuStats{}, + Timestamp: ct.Add(time.Duration(i) * time.Second), + Memory: info.MemoryStats{ + // Memory grows by a KB every second. + WorkingSet: i * 1024, + }, + } + // cpu rate is 1 s/s + s.Cpu.Usage.Total = i * Nanosecond + stats = append(stats, s) + } + cpu, mem := GetPercentiles(stats) + // Cpu mean, max, and 90p should all be 1000 ms/s. + cpuExpected := Percentiles{ + Mean: 1000, + Max: 1000, + Ninety: 1000, + } + if cpu != cpuExpected { + t.Errorf("cpu stats are %+v. Expected %+v", cpu, cpuExpected) + } + memExpected := Percentiles{ + Mean: 50 * 1024, + Max: 99 * 1024, + Ninety: 90 * 1024, + } + if mem != memExpected { + t.Errorf("memory stats are mean %+v. Expected %+v", mem, memExpected) + } +} +func TestSamplesCloseInTimeIgnored(t *testing.T) { + N := uint64(100) + var i uint64 + ct := time.Now() + stats := make([]*info.ContainerStats, 0, N*2) + for i = 1; i < N; i++ { + s1 := &info.ContainerStats{ + Cpu: info.CpuStats{}, + Timestamp: ct.Add(time.Duration(i) * time.Second), + Memory: info.MemoryStats{ + // Memory grows by a KB every second. + WorkingSet: i * 1024, + }, + } + // cpu rate is 1 s/s + s1.Cpu.Usage.Total = i * Nanosecond + stats = append(stats, s1) + + // Add another dummy sample too close in time to the last one. + s2 := &info.ContainerStats{ + Cpu: info.CpuStats{}, + // Add extra millisecond. + Timestamp: ct.Add(time.Duration(i) * time.Second).Add(time.Duration(1) * time.Millisecond), + Memory: info.MemoryStats{ + WorkingSet: i * 1024 * 1024, + }, + } + s2.Cpu.Usage.Total = i * 100 * Nanosecond + stats = append(stats, s2) + } + cpu, mem := GetPercentiles(stats) + // Cpu mean, max, and 90p should all be 1000 ms/s. All high-value samples are discarded. + cpuExpected := Percentiles{ + Mean: 1000, + Max: 1000, + Ninety: 1000, + } + if cpu != cpuExpected { + t.Errorf("cpu stats are %+v. Expected %+v", cpu, cpuExpected) + } + memExpected := Percentiles{ + Mean: 50 * 1024, + Max: 99 * 1024, + Ninety: 90 * 1024, + } + if mem != memExpected { + t.Errorf("memory stats are mean %+v. Expected %+v", mem, memExpected) + } +}