diff --git a/info/container.go b/info/container.go index b3816336..845b6d87 100644 --- a/info/container.go +++ b/info/container.go @@ -464,9 +464,19 @@ type Usage struct { Memory Percentiles `json:"memory"` } +// latest sample collected for a container. +type InstantUsage struct { + // cpu rate in cpu milliseconds/second. + Cpu uint64 `json:"cpu"` + // Memory usage in bytes. + Memory uint64 `json:"memory"` +} + type DerivedStats struct { // Time of generation of these stats. Timestamp time.Time `json:"timestamp"` + // Latest instantaneous sample. + LatestUsage InstantUsage `json:"latest_usage"` // Percentiles in last observed minute. MinuteUsage Usage `json:"minute_usage"` // Percentile in last hour. diff --git a/summary/percentiles.go b/summary/percentiles.go index 01b70b7a..59075c2a 100644 --- a/summary/percentiles.go +++ b/summary/percentiles.go @@ -17,6 +17,7 @@ package summary import ( + "fmt" "math" "sort" @@ -147,23 +148,31 @@ func getPercentComplete(stats []*secondSample) (percent int32) { return } +// Calculate cpurate from two consecutive total cpu usage samples. +func getCpuRate(latest, previous secondSample) (uint64, error) { + var elapsed int64 + elapsed = latest.Timestamp.Sub(previous.Timestamp).Nanoseconds() + if elapsed < 10*milliSecondsToNanoSeconds { + return 0, fmt.Errorf("elapsed time too small: %d ns: time now %s last %s", elapsed, latest.Timestamp.String(), previous.Timestamp.String()) + } + if latest.Cpu < previous.Cpu { + return 0, fmt.Errorf("bad sample: cumulative cpu usage dropped from %d to %d", latest.Cpu, previous.Cpu) + } + // Cpurate is calculated in cpu-milliseconds per second. + cpuRate := (latest.Cpu - previous.Cpu) * secondsToMilliSeconds / uint64(elapsed) + return cpuRate, nil +} + // Returns a percentile sample for a minute by aggregating seconds samples. func GetMinutePercentiles(stats []*secondSample) info.Usage { lastSample := secondSample{} cpu := NewResource(len(stats)) memory := NewResource(len(stats)) for _, stat := range stats { - var elapsed int64 if !lastSample.Timestamp.IsZero() { - elapsed = stat.Timestamp.Sub(lastSample.Timestamp).Nanoseconds() - if elapsed < 10*milliSecondsToNanoSeconds { - glog.Infof("Elapsed time too small: %d ns: time now %s last %s", elapsed, stat.Timestamp.String(), lastSample.Timestamp.String()) - continue - } - glog.V(2).Infof("Read sample: cpu %d, memory %d", stat.Cpu, memory) - cpuRate := (stat.Cpu - lastSample.Cpu) * secondsToMilliSeconds / uint64(elapsed) - if cpuRate < 0 { - glog.Infof("cpu rate too small: %f ns", cpuRate) + cpuRate, err := getCpuRate(*stat, lastSample) + if err != nil { + glog.V(2).Infof("Skipping sample, %v", err) continue } glog.V(2).Infof("Adding cpu rate sample : %d", cpuRate) diff --git a/summary/summary.go b/summary/summary.go index 4d0012fa..8a443b29 100644 --- a/summary/summary.go +++ b/summary/summary.go @@ -47,7 +47,8 @@ type StatsSummary struct { secondSamples []*secondSample // minute percentiles. We track 24 * 60 maximum samples. minuteSamples *SamplesBuffer - // latest derived minute, hour, and day stats. Updated every minute. + // latest derived instant, minute, hour, and day stats. Instant sample updated every second. + // Others updated every minute. derivedStats info.DerivedStats // Guarded by dataLock. dataLock sync.RWMutex } @@ -65,6 +66,7 @@ func (s *StatsSummary) AddSample(stat info.ContainerStats) error { sample.Memory = stat.Memory.WorkingSet } s.secondSamples = append(s.secondSamples, &sample) + s.updateLatestUsage() // TODO(jnagal): Use 'available' to avoid unnecessary computation. if len(s.secondSamples) == 60 { // Make a minute stat. @@ -80,6 +82,29 @@ func (s *StatsSummary) AddSample(stat info.ContainerStats) error { return nil } +func (s *StatsSummary) updateLatestUsage() { + usage := info.InstantUsage{} + numStats := len(s.secondSamples) + if numStats < 1 { + return + } + latest := s.secondSamples[numStats-1] + usage.Memory = latest.Memory + if numStats > 1 { + previous := s.secondSamples[numStats-2] + cpu, err := getCpuRate(*latest, *previous) + if err == nil { + usage.Cpu = cpu + } + } + + s.dataLock.Lock() + defer s.dataLock.Unlock() + s.derivedStats.LatestUsage = usage + s.derivedStats.Timestamp = latest.Timestamp + return +} + // Generate new derived stats based on current minute stats samples. func (s *StatsSummary) updateDerivedStats() error { derived := info.DerivedStats{} @@ -102,6 +127,7 @@ func (s *StatsSummary) updateDerivedStats() error { s.dataLock.Lock() defer s.dataLock.Unlock() + derived.LatestUsage = s.derivedStats.LatestUsage s.derivedStats = derived return nil