prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #663 from vmarmol/limit

Browse Source 
Generalize StatsBuffer into TimedStore
This commit is contained in:
Rohit Jnagal 2015-04-23 12:26:19 -07:00
commit 8197d35ea2
3 changed files with 116 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
storage/memory/memory.go
View File

@ -22,13 +22,14 @@ import (
"github.com/golang/glog" "github.com/golang/glog"
info "github.com/google/cadvisor/info/v1" info "github.com/google/cadvisor/info/v1"
"github.com/google/cadvisor/storage" "github.com/google/cadvisor/storage"
"github.com/google/cadvisor/utils"
) )
// TODO(vmarmol): See about refactoring this class, we have an unecessary redirection of containerStorage and InMemoryStorage. // TODO(vmarmol): See about refactoring this class, we have an unecessary redirection of containerStorage and InMemoryStorage.
// containerStorage is used to store per-container information // containerStorage is used to store per-container information
type containerStorage struct { type containerStorage struct {
ref info.ContainerReference ref info.ContainerReference
recentStats *StatsBuffer recentStats *utils.TimedStore
maxAge time.Duration maxAge time.Duration
lock sync.RWMutex lock sync.RWMutex
} }
@ -38,20 +39,25 @@ func (self *containerStorage) AddStats(stats *info.ContainerStats) error {
defer self.lock.Unlock() defer self.lock.Unlock()
// Add the stat to storage. // Add the stat to storage.
self.recentStats.Add(stats) self.recentStats.Add(stats.Timestamp, stats)
return nil return nil
} }
func (self *containerStorage) RecentStats(start, end time.Time, maxStats int) ([]*info.ContainerStats, error) { func (self *containerStorage) RecentStats(start, end time.Time, maxStats int) ([]*info.ContainerStats, error) {
self.lock.RLock() self.lock.RLock()
defer self.lock.RUnlock() defer self.lock.RUnlock()
return self.recentStats.InTimeRange(start, end, maxStats), nil result := self.recentStats.InTimeRange(start, end, maxStats)
converted := make([]*info.ContainerStats, len(result))
for i, el := range result {
converted[i] = el.(*info.ContainerStats)
}
return converted, nil
} }
func newContainerStore(ref info.ContainerReference, maxAge time.Duration) *containerStorage { func newContainerStore(ref info.ContainerReference, maxAge time.Duration) *containerStorage {
return &containerStorage{ return &containerStorage{
ref: ref, ref: ref,
recentStats: NewStatsBuffer(maxAge), recentStats: utils.NewTimedStore(maxAge),
maxAge: maxAge, maxAge: maxAge,
} }
} }

57
storage/memory/stats_buffer.go → utils/timed_store.go
View File

@ -12,51 +12,57 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package memory package utils
import ( import (
"sort" "sort"
"time" "time"
info "github.com/google/cadvisor/info/v1"
) )
// A time-based buffer for ContainerStats. Holds information for a specific time period. // A time-based buffer for ContainerStats. Holds information for a specific time period.
type StatsBuffer struct { type TimedStore struct {
buffer []*info.ContainerStats buffer []timedStoreData
age time.Duration age time.Duration
} }
// Returns a new thread-compatible StatsBuffer. type timedStoreData struct {
func NewStatsBuffer(age time.Duration) *StatsBuffer { timestamp time.Time
return &StatsBuffer{ data interface{}
buffer: make([]*info.ContainerStats, 0), }
// Returns a new thread-compatible TimedStore.
func NewTimedStore(age time.Duration) *TimedStore {
return &TimedStore{
buffer: make([]timedStoreData, 0),
age: age, age: age,
} }
} }
// Adds an element to the start of the buffer (removing one from the end if necessary). // Adds an element to the start of the buffer (removing one from the end if necessary).
func (self *StatsBuffer) Add(item *info.ContainerStats) { func (self *TimedStore) Add(timestamp time.Time, item interface{}) {
// Remove any elements before the eviction time. // Remove any elements before the eviction time.
evictTime := item.Timestamp.Add(-self.age) evictTime := timestamp.Add(-self.age)
index := sort.Search(len(self.buffer), func(index int) bool { index := sort.Search(len(self.buffer), func(index int) bool {
return self.buffer[index].Timestamp.After(evictTime) return self.buffer[index].timestamp.After(evictTime)
}) })
if index < len(self.buffer) { if index < len(self.buffer) {
self.buffer = self.buffer[index:] self.buffer = self.buffer[index:]
} }
copied := *item copied := item
self.buffer = append(self.buffer, &copied) self.buffer = append(self.buffer, timedStoreData{
timestamp: timestamp,
data: copied,
})
} }
// Returns up to maxResult elements in the specified time period (inclusive). // Returns up to maxResult elements in the specified time period (inclusive).
// Results are from first to last. maxResults of -1 means no limit. When first // Results are from first to last. maxResults of -1 means no limit. When first
// and last are specified, maxResults is ignored. // and last are specified, maxResults is ignored.
func (self *StatsBuffer) InTimeRange(start, end time.Time, maxResults int) []*info.ContainerStats { func (self *TimedStore) InTimeRange(start, end time.Time, maxResults int) []interface{} {
// No stats, return empty. // No stats, return empty.
if len(self.buffer) == 0 { if len(self.buffer) == 0 {
return []*info.ContainerStats{} return []interface{}{}
} }
// Return all results in a time range if specified. // Return all results in a time range if specified.
@ -77,11 +83,11 @@ func (self *StatsBuffer) InTimeRange(start, end time.Time, maxResults int) []*in
// before that element // before that element
startIndex = sort.Search(len(self.buffer), func(index int) bool { startIndex = sort.Search(len(self.buffer), func(index int) bool {
// buffer[index] < start // buffer[index] < start
return self.Get(index).Timestamp.Before(start) return self.getData(index).timestamp.Before(start)
}) - 1 }) - 1
// Check if start is after all the data we have. // Check if start is after all the data we have.
if startIndex < 0 { if startIndex < 0 {
return []*info.ContainerStats{} return []interface{}{}
} }
} }
@ -93,11 +99,11 @@ func (self *StatsBuffer) InTimeRange(start, end time.Time, maxResults int) []*in
// End is the first index smaller than or equal to it (so, not larger). // End is the first index smaller than or equal to it (so, not larger).
endIndex = sort.Search(len(self.buffer), func(index int) bool { endIndex = sort.Search(len(self.buffer), func(index int) bool {
// buffer[index] <= t -> !(buffer[index] > t) // buffer[index] <= t -> !(buffer[index] > t)
return !self.Get(index).Timestamp.After(end) return !self.getData(index).timestamp.After(end)
}) })
// Check if end is before all the data we have. // Check if end is before all the data we have.
if endIndex == len(self.buffer) { if endIndex == len(self.buffer) {
return []*info.ContainerStats{} return []interface{}{}
} }
} }
@ -109,7 +115,7 @@ func (self *StatsBuffer) InTimeRange(start, end time.Time, maxResults int) []*in
} }
// Return in sorted timestamp order so from the "back" to "front". // Return in sorted timestamp order so from the "back" to "front".
result := make([]*info.ContainerStats, numResults) result := make([]interface{}, numResults)
for i := 0; i < numResults; i++ { for i := 0; i < numResults; i++ {
result[i] = self.Get(startIndex - i) result[i] = self.Get(startIndex - i)
} }
@ -117,10 +123,15 @@ func (self *StatsBuffer) InTimeRange(start, end time.Time, maxResults int) []*in
} }
// Gets the element at the specified index. Note that elements are output in LIFO order. // Gets the element at the specified index. Note that elements are output in LIFO order.
func (self *StatsBuffer) Get(index int) *info.ContainerStats { func (self *TimedStore) Get(index int) interface{} {
return self.getData(index).data
}
// Gets the data at the specified index. Note that elements are output in LIFO order.
func (self *TimedStore) getData(index int) timedStoreData {
return self.buffer[len(self.buffer)-index-1] return self.buffer[len(self.buffer)-index-1]
} }
func (self *StatsBuffer) Size() int { func (self *TimedStore) Size() int {
return len(self.buffer) return len(self.buffer)
} }

169
storage/memory/stats_buffer_test.go → utils/timed_store_test.go
View File

@ -12,15 +12,12 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package memory package utils
import ( import (
"strconv"
"strings"
"testing" "testing"
"time" "time"
info "github.com/google/cadvisor/info/v1"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
@ -29,100 +26,78 @@ func createTime(id int) time.Time {
return zero.Add(time.Duration(id+1) * time.Second) return zero.Add(time.Duration(id+1) * time.Second)
} }
func createStats(id int32) *info.ContainerStats { func expectSize(t *testing.T, sb *TimedStore, expectedSize int) {
return &info.ContainerStats{
Timestamp: createTime(int(id)),
Cpu: info.CpuStats{
LoadAverage: id,
},
}
}
func expectSize(t *testing.T, sb *StatsBuffer, expectedSize int) {
if sb.Size() != expectedSize { if sb.Size() != expectedSize {
t.Errorf("Expected size %v, got %v", expectedSize, sb.Size()) t.Errorf("Expected size %v, got %v", expectedSize, sb.Size())
} }
} }
func expectAllElements(t *testing.T, sb *StatsBuffer, expected []int32) { func expectAllElements(t *testing.T, sb *TimedStore, expected []int) {
size := sb.Size() size := sb.Size()
els := make([]*info.ContainerStats, size) els := make([]interface{}, size)
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
els[i] = sb.Get(size - i - 1) els[i] = sb.Get(size - i - 1)
} }
expectElements(t, els, expected) expectElements(t, []interface{}(els), expected)
} }
func getActualElements(actual []*info.ContainerStats) string { func expectElements(t *testing.T, actual []interface{}, expected []int) {
actualElements := make([]string, len(actual))
for i, element := range actual {
actualElements[i] = strconv.Itoa(int(element.Cpu.LoadAverage))
}
return strings.Join(actualElements, ",")
}
func expectElements(t *testing.T, actual []*info.ContainerStats, expected []int32) {
if len(actual) != len(expected) { if len(actual) != len(expected) {
t.Errorf("Expected elements %v, got %v", expected, getActualElements(actual)) t.Errorf("Expected elements %v, got %v", expected, actual)
return return
} }
for i, el := range actual { for i, el := range actual {
if el.Cpu.LoadAverage != expected[i] { if el.(int) != expected[i] {
t.Errorf("Expected elements %v, got %v", expected, getActualElements(actual)) t.Errorf("Expected elements %v, got %v", expected, actual)
return return
} }
} }
} }
func expectElement(t *testing.T, stat *info.ContainerStats, expected int32) {
if stat.Cpu.LoadAverage != expected {
t.Errorf("Expected %d, but received %d", expected, stat.Cpu.LoadAverage)
}
}
func TestAdd(t *testing.T) { func TestAdd(t *testing.T) {
sb := NewStatsBuffer(5 * time.Second) sb := NewTimedStore(5 * time.Second)
// Add 1. // Add 1.
sb.Add(createStats(0)) sb.Add(createTime(0), 0)
expectSize(t, sb, 1) expectSize(t, sb, 1)
expectAllElements(t, sb, []int32{0}) expectAllElements(t, sb, []int{0})
// Fill the buffer. // Fill the buffer.
for i := 1; i <= 5; i++ { for i := 1; i <= 5; i++ {
expectSize(t, sb, i) expectSize(t, sb, i)
sb.Add(createStats(int32(i))) sb.Add(createTime(i), i)
} }
expectSize(t, sb, 5) expectSize(t, sb, 5)
expectAllElements(t, sb, []int32{1, 2, 3, 4, 5}) expectAllElements(t, sb, []int{1, 2, 3, 4, 5})
// Add more than is available in the buffer // Add more than is available in the buffer
sb.Add(createStats(6)) sb.Add(createTime(6), 6)
expectSize(t, sb, 5) expectSize(t, sb, 5)
expectAllElements(t, sb, []int32{2, 3, 4, 5, 6}) expectAllElements(t, sb, []int{2, 3, 4, 5, 6})
// Replace all elements. // Replace all elements.
for i := 7; i <= 10; i++ { for i := 7; i <= 10; i++ {
sb.Add(createStats(int32(i))) sb.Add(createTime(i), i)
} }
expectSize(t, sb, 5) expectSize(t, sb, 5)
expectAllElements(t, sb, []int32{6, 7, 8, 9, 10}) expectAllElements(t, sb, []int{6, 7, 8, 9, 10})
} }
func TestGet(t *testing.T) { func TestGet(t *testing.T) {
sb := NewStatsBuffer(5 * time.Second) sb := NewTimedStore(5 * time.Second)
sb.Add(createStats(1)) sb.Add(createTime(1), 1)
sb.Add(createStats(2)) sb.Add(createTime(2), 2)
sb.Add(createStats(3)) sb.Add(createTime(3), 3)
expectSize(t, sb, 3) expectSize(t, sb, 3)
expectElement(t, sb.Get(0), 3) assert := assert.New(t)
expectElement(t, sb.Get(1), 2) assert.Equal(sb.Get(0).(int), 3)
expectElement(t, sb.Get(2), 1) assert.Equal(sb.Get(1).(int), 2)
assert.Equal(sb.Get(2).(int), 1)
} }
func TestInTimeRange(t *testing.T) { func TestInTimeRange(t *testing.T) {
sb := NewStatsBuffer(5 * time.Second) sb := NewTimedStore(5 * time.Second)
assert := assert.New(t) assert := assert.New(t)
var empty time.Time var empty time.Time
@ -134,60 +109,60 @@ func TestInTimeRange(t *testing.T) {
assert.Empty(sb.InTimeRange(empty, empty, 10)) assert.Empty(sb.InTimeRange(empty, empty, 10))
// One element. // One element.
sb.Add(createStats(1)) sb.Add(createTime(1), 1)
expectSize(t, sb, 1) expectSize(t, sb, 1)
expectElements(t, sb.InTimeRange(createTime(0), createTime(5), 10), []int32{1}) expectElements(t, sb.InTimeRange(createTime(0), createTime(5), 10), []int{1})
expectElements(t, sb.InTimeRange(createTime(1), createTime(5), 10), []int32{1}) expectElements(t, sb.InTimeRange(createTime(1), createTime(5), 10), []int{1})
expectElements(t, sb.InTimeRange(createTime(0), createTime(1), 10), []int32{1}) expectElements(t, sb.InTimeRange(createTime(0), createTime(1), 10), []int{1})
expectElements(t, sb.InTimeRange(createTime(1), createTime(1), 10), []int32{1}) expectElements(t, sb.InTimeRange(createTime(1), createTime(1), 10), []int{1})
assert.Empty(sb.InTimeRange(createTime(2), createTime(5), 10)) assert.Empty(sb.InTimeRange(createTime(2), createTime(5), 10))
// Two element. // Two element.
sb.Add(createStats(2)) sb.Add(createTime(2), 2)
expectSize(t, sb, 2) expectSize(t, sb, 2)
expectElements(t, sb.InTimeRange(createTime(0), createTime(5), 10), []int32{1, 2}) expectElements(t, sb.InTimeRange(createTime(0), createTime(5), 10), []int{1, 2})
expectElements(t, sb.InTimeRange(createTime(1), createTime(5), 10), []int32{1, 2}) expectElements(t, sb.InTimeRange(createTime(1), createTime(5), 10), []int{1, 2})
expectElements(t, sb.InTimeRange(createTime(0), createTime(2), 10), []int32{1, 2}) expectElements(t, sb.InTimeRange(createTime(0), createTime(2), 10), []int{1, 2})
expectElements(t, sb.InTimeRange(createTime(1), createTime(2), 10), []int32{1, 2}) expectElements(t, sb.InTimeRange(createTime(1), createTime(2), 10), []int{1, 2})
expectElements(t, sb.InTimeRange(createTime(1), createTime(1), 10), []int32{1}) expectElements(t, sb.InTimeRange(createTime(1), createTime(1), 10), []int{1})
expectElements(t, sb.InTimeRange(createTime(2), createTime(2), 10), []int32{2}) expectElements(t, sb.InTimeRange(createTime(2), createTime(2), 10), []int{2})
assert.Empty(sb.InTimeRange(createTime(3), createTime(5), 10)) assert.Empty(sb.InTimeRange(createTime(3), createTime(5), 10))
// Many elements. // Many elements.
sb.Add(createStats(3)) sb.Add(createTime(3), 3)
sb.Add(createStats(4)) sb.Add(createTime(4), 4)
expectSize(t, sb, 4) expectSize(t, sb, 4)
expectElements(t, sb.InTimeRange(createTime(0), createTime(5), 10), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(createTime(0), createTime(5), 10), []int{1, 2, 3, 4})
expectElements(t, sb.InTimeRange(createTime(0), createTime(5), 10), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(createTime(0), createTime(5), 10), []int{1, 2, 3, 4})
expectElements(t, sb.InTimeRange(createTime(1), createTime(5), 10), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(createTime(1), createTime(5), 10), []int{1, 2, 3, 4})
expectElements(t, sb.InTimeRange(createTime(0), createTime(4), 10), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(createTime(0), createTime(4), 10), []int{1, 2, 3, 4})
expectElements(t, sb.InTimeRange(createTime(1), createTime(4), 10), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(createTime(1), createTime(4), 10), []int{1, 2, 3, 4})
expectElements(t, sb.InTimeRange(createTime(0), createTime(2), 10), []int32{1, 2}) expectElements(t, sb.InTimeRange(createTime(0), createTime(2), 10), []int{1, 2})
expectElements(t, sb.InTimeRange(createTime(1), createTime(2), 10), []int32{1, 2}) expectElements(t, sb.InTimeRange(createTime(1), createTime(2), 10), []int{1, 2})
expectElements(t, sb.InTimeRange(createTime(2), createTime(3), 10), []int32{2, 3}) expectElements(t, sb.InTimeRange(createTime(2), createTime(3), 10), []int{2, 3})
expectElements(t, sb.InTimeRange(createTime(3), createTime(4), 10), []int32{3, 4}) expectElements(t, sb.InTimeRange(createTime(3), createTime(4), 10), []int{3, 4})
expectElements(t, sb.InTimeRange(createTime(3), createTime(5), 10), []int32{3, 4}) expectElements(t, sb.InTimeRange(createTime(3), createTime(5), 10), []int{3, 4})
assert.Empty(sb.InTimeRange(createTime(5), createTime(5), 10)) assert.Empty(sb.InTimeRange(createTime(5), createTime(5), 10))
// Start and end time ignores maxResults. // Start and end time ignores maxResults.
expectElements(t, sb.InTimeRange(createTime(1), createTime(5), 1), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(createTime(1), createTime(5), 1), []int{1, 2, 3, 4})
// No start time. // No start time.
expectElements(t, sb.InTimeRange(empty, createTime(5), 10), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(empty, createTime(5), 10), []int{1, 2, 3, 4})
expectElements(t, sb.InTimeRange(empty, createTime(4), 10), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(empty, createTime(4), 10), []int{1, 2, 3, 4})
expectElements(t, sb.InTimeRange(empty, createTime(3), 10), []int32{1, 2, 3}) expectElements(t, sb.InTimeRange(empty, createTime(3), 10), []int{1, 2, 3})
expectElements(t, sb.InTimeRange(empty, createTime(2), 10), []int32{1, 2}) expectElements(t, sb.InTimeRange(empty, createTime(2), 10), []int{1, 2})
expectElements(t, sb.InTimeRange(empty, createTime(1), 10), []int32{1}) expectElements(t, sb.InTimeRange(empty, createTime(1), 10), []int{1})
// No end time. // No end time.
expectElements(t, sb.InTimeRange(createTime(0), empty, 10), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(createTime(0), empty, 10), []int{1, 2, 3, 4})
expectElements(t, sb.InTimeRange(createTime(1), empty, 10), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(createTime(1), empty, 10), []int{1, 2, 3, 4})
expectElements(t, sb.InTimeRange(createTime(2), empty, 10), []int32{2, 3, 4}) expectElements(t, sb.InTimeRange(createTime(2), empty, 10), []int{2, 3, 4})
expectElements(t, sb.InTimeRange(createTime(3), empty, 10), []int32{3, 4}) expectElements(t, sb.InTimeRange(createTime(3), empty, 10), []int{3, 4})
expectElements(t, sb.InTimeRange(createTime(4), empty, 10), []int32{4}) expectElements(t, sb.InTimeRange(createTime(4), empty, 10), []int{4})
// No start or end time. // No start or end time.
expectElements(t, sb.InTimeRange(empty, empty, 10), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(empty, empty, 10), []int{1, 2, 3, 4})
// Start after data. // Start after data.
assert.Empty(sb.InTimeRange(createTime(5), createTime(5), 10)) assert.Empty(sb.InTimeRange(createTime(5), createTime(5), 10))
@ -199,19 +174,19 @@ func TestInTimeRange(t *testing.T) {
} }
func TestInTimeRangeWithLimit(t *testing.T) { func TestInTimeRangeWithLimit(t *testing.T) {
sb := NewStatsBuffer(5 * time.Second) sb := NewTimedStore(5 * time.Second)
sb.Add(createStats(1)) sb.Add(createTime(1), 1)
sb.Add(createStats(2)) sb.Add(createTime(2), 2)
sb.Add(createStats(3)) sb.Add(createTime(3), 3)
sb.Add(createStats(4)) sb.Add(createTime(4), 4)
expectSize(t, sb, 4) expectSize(t, sb, 4)
var empty time.Time var empty time.Time
// Limit cuts off from latest timestamp. // Limit cuts off from latest timestamp.
expectElements(t, sb.InTimeRange(empty, empty, 4), []int32{1, 2, 3, 4}) expectElements(t, sb.InTimeRange(empty, empty, 4), []int{1, 2, 3, 4})
expectElements(t, sb.InTimeRange(empty, empty, 3), []int32{2, 3, 4}) expectElements(t, sb.InTimeRange(empty, empty, 3), []int{2, 3, 4})
expectElements(t, sb.InTimeRange(empty, empty, 2), []int32{3, 4}) expectElements(t, sb.InTimeRange(empty, empty, 2), []int{3, 4})
expectElements(t, sb.InTimeRange(empty, empty, 1), []int32{4}) expectElements(t, sb.InTimeRange(empty, empty, 1), []int{4})
assert.Empty(t, sb.InTimeRange(empty, empty, 0)) assert.Empty(t, sb.InTimeRange(empty, empty, 0))
} }