prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

remove Samples() Percentiles() from bigquery storage driver

Browse Source
This commit is contained in:
Nan Monnand Deng 2014-09-05 15:35:31 -04:00
parent 9d6235f4d0
commit 260625f421
3 changed files with 2 additions and 270 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
storage/bigquery/bigquery.go
View File

@ -17,7 +17,6 @@ package bigquery
import ( import (
"fmt" "fmt"
"strconv" "strconv"
"strings"
"time" "time"
bigquery "code.google.com/p/google-api-go-client/bigquery/v2" bigquery "code.google.com/p/google-api-go-client/bigquery/v2"
@ -437,25 +436,7 @@ func (self *bigqueryStorage) RecentStats(containerName string, numStats int) ([]
} }
func (self *bigqueryStorage) Samples(containerName string, numSamples int) ([]*info.ContainerStatsSample, error) { func (self *bigqueryStorage) Samples(containerName string, numSamples int) ([]*info.ContainerStatsSample, error) {
if numSamples == 0 { panic("will be removed")
return nil, nil
}
header, rows, err := self.getRecentRows(containerName, numSamples)
if err != nil {
return nil, err
}
sampleList := make([]*info.ContainerStatsSample, 0, len(rows))
for _, row := range rows {
sample, err := self.valuesToContainerSample(header, row)
if err != nil {
return nil, err
}
if sample == nil {
continue
}
sampleList = append(sampleList, sample)
}
return sampleList, nil
} }
func (self *bigqueryStorage) Close() error { func (self *bigqueryStorage) Close() error {
@ -469,74 +450,7 @@ func (self *bigqueryStorage) Percentiles(
cpuUsagePercentiles []int, cpuUsagePercentiles []int,
memUsagePercentiles []int, memUsagePercentiles []int,
) (*info.ContainerStatsPercentiles, error) { ) (*info.ContainerStatsPercentiles, error) {
selectedCol := make([]string, 0, len(cpuUsagePercentiles)+len(memUsagePercentiles)+1) panic("will be removed")
selectedCol = append(selectedCol, fmt.Sprintf("max(%v)", colMemoryUsage))
for _, p := range cpuUsagePercentiles {
selectedCol = append(selectedCol, fmt.Sprintf("percentile(%v, %v)", colCpuInstantUsage, p))
}
for _, p := range memUsagePercentiles {
selectedCol = append(selectedCol, fmt.Sprintf("percentile(%v, %v)", colMemoryUsage, p))
}
tableName, err := self.client.GetTableName()
if err != nil {
return nil, err
}
query := fmt.Sprintf("SELECT %v FROM %v WHERE %v='%v' AND %v='%v' AND timestamp > DATE_ADD(CURRENT_TIMESTAMP(), -%v, 'SECOND')",
strings.Join(selectedCol, ","),
tableName,
colContainerName,
containerName,
colMachineName,
self.machineName,
self.windowLen.Seconds(),
)
_, rows, err := self.client.Query(query)
if err != nil {
return nil, err
}
if len(rows) != 1 {
return nil, nil
}
point := rows[0]
ret := new(info.ContainerStatsPercentiles)
ret.MaxMemoryUsage, err = convertToUint64(point[0])
if err != nil {
return nil, fmt.Errorf("invalid max memory usage: %v", err)
}
retrievedCpuPercentiles := point[1 : 1+len(cpuUsagePercentiles)]
for i, p := range cpuUsagePercentiles {
v, err := convertToUint64(retrievedCpuPercentiles[i])
if err != nil {
return nil, fmt.Errorf("invalid cpu usage: %v", err)
}
ret.CpuUsagePercentiles = append(
ret.CpuUsagePercentiles,
info.Percentile{
Percentage: p,
Value: v,
},
)
}
retrievedMemoryPercentiles := point[1+len(cpuUsagePercentiles):]
for i, p := range memUsagePercentiles {
v, err := convertToUint64(retrievedMemoryPercentiles[i])
if err != nil {
return nil, fmt.Errorf("invalid memory usage: %v", err)
}
ret.MemoryUsagePercentiles = append(
ret.MemoryUsagePercentiles,
info.Percentile{
Percentage: p,
Value: v,
},
)
}
return ret, nil
} }
// Create a new bigquery storage driver. // Create a new bigquery storage driver.

74
storage/cache/memcache.go vendored
View File

@ -1,74 +0,0 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cache
import (
"github.com/google/cadvisor/info"
"github.com/google/cadvisor/storage"
"github.com/google/cadvisor/storage/memory"
)
type cachedStorageDriver struct {
maxNumStatsInCache int
maxNumSamplesInCache int
cache storage.StorageDriver
backend storage.StorageDriver
}
func (self *cachedStorageDriver) AddStats(ref info.ContainerReference, stats *info.ContainerStats) error {
err := self.cache.AddStats(ref, stats)
if err != nil {
return err
}
err = self.backend.AddStats(ref, stats)
if err != nil {
return err
}
return nil
}
func (self *cachedStorageDriver) RecentStats(containerName string, numStats int) ([]*info.ContainerStats, error) {
if numStats <= self.maxNumStatsInCache {
return self.cache.RecentStats(containerName, numStats)
}
return self.backend.RecentStats(containerName, numStats)
}
// TODO(vishh): Calculate percentiles from cached stats instead of reaching the DB. This will make the UI truly independent of the backend storage.
func (self *cachedStorageDriver) Percentiles(containerName string, cpuUsagePercentiles []int, memUsagePercentiles []int) (*info.ContainerStatsPercentiles, error) {
return self.backend.Percentiles(containerName, cpuUsagePercentiles, memUsagePercentiles)
}
func (self *cachedStorageDriver) Samples(containerName string, numSamples int) ([]*info.ContainerStatsSample, error) {
if numSamples <= self.maxNumSamplesInCache {
return self.cache.Samples(containerName, numSamples)
}
return self.backend.Samples(containerName, numSamples)
}
func (self *cachedStorageDriver) Close() error {
self.cache.Close()
return self.backend.Close()
}
// TODO(vishh): Cache all samples for a given duration and do not cap the maximum number of samples. This is useful if we happen to change the housekeeping duration.
func MemoryCache(maxNumSamplesInCache, maxNumStatsInCache int, driver storage.StorageDriver) storage.StorageDriver {
return &cachedStorageDriver{
maxNumStatsInCache: maxNumStatsInCache,
maxNumSamplesInCache: maxNumSamplesInCache,
cache: memory.New(maxNumSamplesInCache, maxNumStatsInCache),
backend: driver,
}
}

108
storage/cache/memcache_test.go vendored
View File

@ -1,108 +0,0 @@
// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cache
import (
"testing"
"github.com/google/cadvisor/info"
"github.com/google/cadvisor/storage"
"github.com/google/cadvisor/storage/memory"
"github.com/google/cadvisor/storage/test"
)
type cacheTestStorageDriver struct {
base storage.StorageDriver
}
func (self *cacheTestStorageDriver) StatsEq(a, b *info.ContainerStats) bool {
return test.DefaultStatsEq(a, b)
}
func (self *cacheTestStorageDriver) AddStats(ref info.ContainerReference, stats *info.ContainerStats) error {
return self.base.AddStats(ref, stats)
}
func (self *cacheTestStorageDriver) RecentStats(containerName string, numStats int) ([]*info.ContainerStats, error) {
return self.base.RecentStats(containerName, numStats)
}
func (self *cacheTestStorageDriver) Percentiles(containerName string, cpuUsagePercentiles []int, memUsagePercentiles []int) (*info.ContainerStatsPercentiles, error) {
return self.base.Percentiles(containerName, cpuUsagePercentiles, memUsagePercentiles)
}
func (self *cacheTestStorageDriver) Samples(containerName string, numSamples int) ([]*info.ContainerStatsSample, error) {
return self.base.Samples(containerName, numSamples)
}
func (self *cacheTestStorageDriver) Close() error {
return self.base.Close()
}
func runStorageTest(f func(test.TestStorageDriver, *testing.T), t *testing.T) {
maxSize := 200
for N := 10; N < maxSize; N += 10 {
testDriver := &cacheTestStorageDriver{}
backend := memory.New(N*2, N*2)
testDriver.base = MemoryCache(N, N, backend)
f(testDriver, t)
}
}
func TestMaxMemoryUsage(t *testing.T) {
runStorageTest(test.StorageDriverTestMaxMemoryUsage, t)
}
func TestSampleCpuUsage(t *testing.T) {
runStorageTest(test.StorageDriverTestSampleCpuUsage, t)
}
func TestSamplesWithoutSample(t *testing.T) {
runStorageTest(test.StorageDriverTestSamplesWithoutSample, t)
}
func TestPercentilesWithoutSample(t *testing.T) {
runStorageTest(test.StorageDriverTestPercentilesWithoutSample, t)
}
func TestPercentiles(t *testing.T) {
N := 100
testDriver := &cacheTestStorageDriver{}
backend := memory.New(N*2, N*2)
testDriver.base = MemoryCache(N, N, backend)
test.StorageDriverTestPercentiles(testDriver, t)
}
func TestRetrievePartialRecentStats(t *testing.T) {
runStorageTest(test.StorageDriverTestRetrievePartialRecentStats, t)
}
func TestRetrieveAllRecentStats(t *testing.T) {
runStorageTest(test.StorageDriverTestRetrieveAllRecentStats, t)
}
func TestNoRecentStats(t *testing.T) {
runStorageTest(test.StorageDriverTestNoRecentStats, t)
}
func TestNoSamples(t *testing.T) {
runStorageTest(test.StorageDriverTestNoSamples, t)
}
func TestPercentilesWithoutStats(t *testing.T) {
runStorageTest(test.StorageDriverTestPercentilesWithoutStats, t)
}