prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #661 from vmarmol/limit

Browse Source 
Move StatsBuffer to be time-based
This commit is contained in:
Rohit Jnagal 2015-04-22 09:51:04 -07:00
commit a32015579e
7 changed files with 60 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/runtime_options.md
View File

@ -2,6 +2,14 @@
This document describes a set of runtime flags available in cAdvisor.
## Local Storage Duration
cAdvisor stores the latest historical data in memory. How long of a history it stores can be configured with the `--storage_duration` flag.
```
--storage_duration: How long to store data.
```
## Housekeeping
Housekeeping is the periodic actions cAdvisor takes. During these actions, cAdvisor will gather container stats. These flags control how and when cAdvisor performs housekeeping.

2
manager/manager_test.go
View File

@ -81,7 +81,7 @@ func expectManagerWithContainers(containers []string, query *info.ContainerInfoR
infosMap[container] = itest.GenerateRandomContainerInfo(container, 4, query, 1*time.Second)
}
memoryStorage := memory.New(query.NumStats, nil)
memoryStorage := memory.New(time.Duration(query.NumStats)*time.Second, nil)
sysfs := &fakesysfs.FakeSysFs{}
m := createManagerAndAddContainers(
memoryStorage,

16
storage/memory/memory.go
View File

@ -29,7 +29,7 @@ import (
type containerStorage struct {
ref info.ContainerReference
recentStats *StatsBuffer
maxNumStats int
maxAge time.Duration
lock sync.RWMutex
}
@ -48,18 +48,18 @@ func (self *containerStorage) RecentStats(start, end time.Time, maxStats int) ([
return self.recentStats.InTimeRange(start, end, maxStats), nil
}
func newContainerStore(ref info.ContainerReference, maxNumStats int) *containerStorage {
func newContainerStore(ref info.ContainerReference, maxAge time.Duration) *containerStorage {
return &containerStorage{
ref: ref,
recentStats: NewStatsBuffer(maxNumStats),
maxNumStats: maxNumStats,
recentStats: NewStatsBuffer(maxAge),
maxAge: maxAge,
}
}
type InMemoryStorage struct {
lock sync.RWMutex
containerStorageMap map[string]*containerStorage
maxNumStats int
maxAge time.Duration
backend storage.StorageDriver
}
@ -71,7 +71,7 @@ func (self *InMemoryStorage) AddStats(ref info.ContainerReference, stats *info.C
self.lock.Lock()
defer self.lock.Unlock()
if cstore, ok = self.containerStorageMap[ref.Name]; !ok {
cstore = newContainerStore(ref, self.maxNumStats)
cstore = newContainerStore(ref, self.maxAge)
self.containerStorageMap[ref.Name] = cstore
}
}()
@ -113,12 +113,12 @@ func (self *InMemoryStorage) Close() error {
}
func New(
maxNumStats int,
maxAge time.Duration,
backend storage.StorageDriver,
) *InMemoryStorage {
ret := &InMemoryStorage{
containerStorageMap: make(map[string]*containerStorage, 32),
maxNumStats: maxNumStats,
maxAge: maxAge,
backend: backend,
}
return ret

4
storage/memory/memory_test.go
View File

@ -47,7 +47,7 @@ func getRecentStats(t *testing.T, memoryStorage *InMemoryStorage, numStats int)
}
func TestAddStats(t *testing.T) {
memoryStorage := New(60, nil)
memoryStorage := New(60*time.Second, nil)
assert := assert.New(t)
assert.Nil(memoryStorage.AddStats(containerRef, makeStat(0)))
@ -70,7 +70,7 @@ func TestRecentStatsNoRecentStats(t *testing.T) {
// Make an instance of InMemoryStorage with n stats.
func makeWithStats(n int) *InMemoryStorage {
memoryStorage := New(60, nil)
memoryStorage := New(60*time.Second, nil)
for i := 0; i < n; i++ {
memoryStorage.AddStats(containerRef, makeStat(i))

45
storage/memory/stats_buffer.go
View File

@ -21,30 +21,33 @@ import (
info "github.com/google/cadvisor/info/v1"
)
// A circular buffer for ContainerStats.
// A time-based buffer for ContainerStats. Holds information for a specific time period.
type StatsBuffer struct {
buffer []*info.ContainerStats
size int
index int
age time.Duration
}
// Returns a new thread-compatible StatsBuffer.
func NewStatsBuffer(size int) *StatsBuffer {
func NewStatsBuffer(age time.Duration) *StatsBuffer {
return &StatsBuffer{
buffer: make([]*info.ContainerStats, size),
size: 0,
index: size - 1,
buffer: make([]*info.ContainerStats, 0),
age: age,
}
}
// Adds an element to the start of the buffer (removing one from the end if necessary).
func (self *StatsBuffer) Add(item *info.ContainerStats) {
if self.size < len(self.buffer) {
self.size++
// Remove any elements before the eviction time.
evictTime := item.Timestamp.Add(-self.age)
index := sort.Search(len(self.buffer), func(index int) bool {
return self.buffer[index].Timestamp.After(evictTime)
})
if index < len(self.buffer) {
self.buffer = self.buffer[index:]
}
self.index = (self.index + 1) % len(self.buffer)
copied := *item
self.buffer[self.index] = &copied
self.buffer = append(self.buffer, &copied)
}
// Returns up to maxResult elements in the specified time period (inclusive).
@ -52,7 +55,7 @@ func (self *StatsBuffer) Add(item *info.ContainerStats) {
// and last are specified, maxResults is ignored.
func (self *StatsBuffer) InTimeRange(start, end time.Time, maxResults int) []*info.ContainerStats {
// No stats, return empty.
if self.size == 0 {
if len(self.buffer) == 0 {
return []*info.ContainerStats{}
}
@ -67,12 +70,12 @@ func (self *StatsBuffer) InTimeRange(start, end time.Time, maxResults int) []*in
var startIndex int
if start.IsZero() {
// None specified, start at the beginning.
startIndex = self.size - 1
startIndex = len(self.buffer) - 1
} else {
// Start is the index before the elements smaller than it. We do this by
// finding the first element smaller than start and taking the index
// before that element
startIndex = sort.Search(self.size, func(index int) bool {
startIndex = sort.Search(len(self.buffer), func(index int) bool {
// buffer[index] < start
return self.Get(index).Timestamp.Before(start)
}) - 1
@ -88,12 +91,12 @@ func (self *StatsBuffer) InTimeRange(start, end time.Time, maxResults int) []*in
endIndex = 0
} else {
// End is the first index smaller than or equal to it (so, not larger).
endIndex = sort.Search(self.size, func(index int) bool {
endIndex = sort.Search(len(self.buffer), func(index int) bool {
// buffer[index] <= t -> !(buffer[index] > t)
return !self.Get(index).Timestamp.After(end)
})
// Check if end is before all the data we have.
if endIndex == self.size {
if endIndex == len(self.buffer) {
return []*info.ContainerStats{}
}
}
@ -113,15 +116,11 @@ func (self *StatsBuffer) InTimeRange(start, end time.Time, maxResults int) []*in
return result
}
// Gets the element at the specified index. Note that elements are stored in LIFO order.
// Gets the element at the specified index. Note that elements are output in LIFO order.
func (self *StatsBuffer) Get(index int) *info.ContainerStats {
calculatedIndex := self.index - index
if calculatedIndex < 0 {
calculatedIndex += len(self.buffer)
}
return self.buffer[calculatedIndex]
return self.buffer[len(self.buffer)-index-1]
}
func (self *StatsBuffer) Size() int {
return self.size
return len(self.buffer)
}

28
storage/memory/stats_buffer_test.go
View File

@ -53,18 +53,22 @@ func expectAllElements(t *testing.T, sb *StatsBuffer, expected []int32) {
expectElements(t, els, expected)
}
func getActualElements(actual []*info.ContainerStats) string {
actualElements := make([]string, len(actual))
for i, element := range actual {
actualElements[i] = strconv.Itoa(int(element.Cpu.LoadAverage))
}
return strings.Join(actualElements, ",")
}
func expectElements(t *testing.T, actual []*info.ContainerStats, expected []int32) {
if len(actual) != len(expected) {
t.Errorf("Expected elements %v, got %v", expected, actual)
t.Errorf("Expected elements %v, got %v", expected, getActualElements(actual))
return
}
for i, el := range actual {
if el.Cpu.LoadAverage != expected[i] {
actualElements := make([]string, len(actual))
for i, element := range actual {
actualElements[i] = strconv.Itoa(int(element.Cpu.LoadAverage))
}
t.Errorf("Expected elements %v, got %v", expected, strings.Join(actualElements, ","))
t.Errorf("Expected elements %v, got %v", expected, getActualElements(actual))
return
}
}
@ -77,12 +81,12 @@ func expectElement(t *testing.T, stat *info.ContainerStats, expected int32) {
}
func TestAdd(t *testing.T) {
sb := NewStatsBuffer(5)
sb := NewStatsBuffer(5 * time.Second)
// Add 1.
sb.Add(createStats(1))
sb.Add(createStats(0))
expectSize(t, sb, 1)
expectAllElements(t, sb, []int32{1})
expectAllElements(t, sb, []int32{0})
// Fill the buffer.
for i := 1; i <= 5; i++ {
@ -106,7 +110,7 @@ func TestAdd(t *testing.T) {
}
func TestGet(t *testing.T) {
sb := NewStatsBuffer(5)
sb := NewStatsBuffer(5 * time.Second)
sb.Add(createStats(1))
sb.Add(createStats(2))
sb.Add(createStats(3))
@ -118,7 +122,7 @@ func TestGet(t *testing.T) {
}
func TestInTimeRange(t *testing.T) {
sb := NewStatsBuffer(5)
sb := NewStatsBuffer(5 * time.Second)
assert := assert.New(t)
var empty time.Time
@ -195,7 +199,7 @@ func TestInTimeRange(t *testing.T) {
}
func TestInTimeRangeWithLimit(t *testing.T) {
sb := NewStatsBuffer(5)
sb := NewStatsBuffer(5 * time.Second)
sb.Add(createStats(1))
sb.Add(createStats(2))
sb.Add(createStats(3))

14
storagedriver.go
View File

@ -21,7 +21,6 @@ import (
"time"
"github.com/golang/glog"
"github.com/google/cadvisor/manager"
"github.com/google/cadvisor/storage"
"github.com/google/cadvisor/storage/bigquery"
"github.com/google/cadvisor/storage/influxdb"
@ -35,20 +34,13 @@ var argDbName = flag.String("storage_driver_db", "cadvisor", "database name")
var argDbTable = flag.String("storage_driver_table", "stats", "table name")
var argDbIsSecure = flag.Bool("storage_driver_secure", false, "use secure connection with database")
var argDbBufferDuration = flag.Duration("storage_driver_buffer_duration", 60*time.Second, "Writes in the storage driver will be buffered for this duration, and committed to the non memory backends as a single transaction")
const statsRequestedByUI = 60
var storageDuration = flag.Duration("storage_duration", 2*time.Minute, "How long to keep data stored (Default: 2min).")
// Creates a memory storage with an optional backend storage option.
func NewMemoryStorage(backendStorageName string) (*memory.InMemoryStorage, error) {
var storageDriver *memory.InMemoryStorage
var backendStorage storage.StorageDriver
var err error
// TODO(vmarmol): We shouldn't need the housekeeping interval here and it shouldn't be public.
statsToCache := int(*argDbBufferDuration / *manager.HousekeepingInterval)
if statsToCache < statsRequestedByUI {
// The UI requests the most recent 60 stats by default.
statsToCache = statsRequestedByUI
}
switch backendStorageName {
case "":
backendStorage = nil
@ -91,7 +83,7 @@ func NewMemoryStorage(backendStorageName string) (*memory.InMemoryStorage, error
} else {
glog.Infof("No backend storage selected")
}
glog.Infof("Caching %d stats in memory", statsToCache)
storageDriver = memory.New(statsToCache, backendStorage)
glog.Infof("Caching stats in memory for %v", *storageDuration)
storageDriver = memory.New(*storageDuration, backendStorage)
return storageDriver, nil
}