remove Samples() Percentiles() from bigquery storage driver
This commit is contained in:
Nan Monnand Deng
parent
9d6235f4d0
commit
260625f421
@ -17,7 +17,6 @@ package bigquery
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
bigquery "code.google.com/p/google-api-go-client/bigquery/v2"
|
||||
@ -437,25 +436,7 @@ func (self *bigqueryStorage) RecentStats(containerName string, numStats int) ([]
|
||||
}
|
||||
|
||||
func (self *bigqueryStorage) Samples(containerName string, numSamples int) ([]*info.ContainerStatsSample, error) {
|
||||
if numSamples == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
header, rows, err := self.getRecentRows(containerName, numSamples)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sampleList := make([]*info.ContainerStatsSample, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
sample, err := self.valuesToContainerSample(header, row)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if sample == nil {
|
||||
continue
|
||||
}
|
||||
sampleList = append(sampleList, sample)
|
||||
}
|
||||
return sampleList, nil
|
||||
panic("will be removed")
|
||||
}
|
||||
|
||||
func (self *bigqueryStorage) Close() error {
|
||||
@ -469,74 +450,7 @@ func (self *bigqueryStorage) Percentiles(
|
||||
cpuUsagePercentiles []int,
|
||||
memUsagePercentiles []int,
|
||||
) (*info.ContainerStatsPercentiles, error) {
|
||||
selectedCol := make([]string, 0, len(cpuUsagePercentiles)+len(memUsagePercentiles)+1)
|
||||
|
||||
selectedCol = append(selectedCol, fmt.Sprintf("max(%v)", colMemoryUsage))
|
||||
for _, p := range cpuUsagePercentiles {
|
||||
selectedCol = append(selectedCol, fmt.Sprintf("percentile(%v, %v)", colCpuInstantUsage, p))
|
||||
}
|
||||
for _, p := range memUsagePercentiles {
|
||||
selectedCol = append(selectedCol, fmt.Sprintf("percentile(%v, %v)", colMemoryUsage, p))
|
||||
}
|
||||
|
||||
tableName, err := self.client.GetTableName()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
query := fmt.Sprintf("SELECT %v FROM %v WHERE %v='%v' AND %v='%v' AND timestamp > DATE_ADD(CURRENT_TIMESTAMP(), -%v, 'SECOND')",
|
||||
strings.Join(selectedCol, ","),
|
||||
tableName,
|
||||
colContainerName,
|
||||
containerName,
|
||||
colMachineName,
|
||||
self.machineName,
|
||||
self.windowLen.Seconds(),
|
||||
)
|
||||
_, rows, err := self.client.Query(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(rows) != 1 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
point := rows[0]
|
||||
|
||||
ret := new(info.ContainerStatsPercentiles)
|
||||
ret.MaxMemoryUsage, err = convertToUint64(point[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid max memory usage: %v", err)
|
||||
}
|
||||
retrievedCpuPercentiles := point[1 : 1+len(cpuUsagePercentiles)]
|
||||
for i, p := range cpuUsagePercentiles {
|
||||
v, err := convertToUint64(retrievedCpuPercentiles[i])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid cpu usage: %v", err)
|
||||
}
|
||||
ret.CpuUsagePercentiles = append(
|
||||
ret.CpuUsagePercentiles,
|
||||
info.Percentile{
|
||||
Percentage: p,
|
||||
Value: v,
|
||||
},
|
||||
)
|
||||
}
|
||||
retrievedMemoryPercentiles := point[1+len(cpuUsagePercentiles):]
|
||||
for i, p := range memUsagePercentiles {
|
||||
v, err := convertToUint64(retrievedMemoryPercentiles[i])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid memory usage: %v", err)
|
||||
}
|
||||
ret.MemoryUsagePercentiles = append(
|
||||
ret.MemoryUsagePercentiles,
|
||||
info.Percentile{
|
||||
Percentage: p,
|
||||
Value: v,
|
||||
},
|
||||
)
|
||||
}
|
||||
return ret, nil
|
||||
panic("will be removed")
|
||||
}
|
||||
|
||||
// Create a new bigquery storage driver.
|
||||
|
||||
74
storage/cache/memcache.go
vendored
74
storage/cache/memcache.go
vendored
@ -1,74 +0,0 @@
|
||||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"github.com/google/cadvisor/info"
|
||||
"github.com/google/cadvisor/storage"
|
||||
"github.com/google/cadvisor/storage/memory"
|
||||
)
|
||||
|
||||
type cachedStorageDriver struct {
|
||||
maxNumStatsInCache int
|
||||
maxNumSamplesInCache int
|
||||
cache storage.StorageDriver
|
||||
backend storage.StorageDriver
|
||||
}
|
||||
|
||||
func (self *cachedStorageDriver) AddStats(ref info.ContainerReference, stats *info.ContainerStats) error {
|
||||
err := self.cache.AddStats(ref, stats)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = self.backend.AddStats(ref, stats)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (self *cachedStorageDriver) RecentStats(containerName string, numStats int) ([]*info.ContainerStats, error) {
|
||||
if numStats <= self.maxNumStatsInCache {
|
||||
return self.cache.RecentStats(containerName, numStats)
|
||||
}
|
||||
return self.backend.RecentStats(containerName, numStats)
|
||||
}
|
||||
|
||||
// TODO(vishh): Calculate percentiles from cached stats instead of reaching the DB. This will make the UI truly independent of the backend storage.
|
||||
func (self *cachedStorageDriver) Percentiles(containerName string, cpuUsagePercentiles []int, memUsagePercentiles []int) (*info.ContainerStatsPercentiles, error) {
|
||||
return self.backend.Percentiles(containerName, cpuUsagePercentiles, memUsagePercentiles)
|
||||
}
|
||||
|
||||
func (self *cachedStorageDriver) Samples(containerName string, numSamples int) ([]*info.ContainerStatsSample, error) {
|
||||
if numSamples <= self.maxNumSamplesInCache {
|
||||
return self.cache.Samples(containerName, numSamples)
|
||||
}
|
||||
return self.backend.Samples(containerName, numSamples)
|
||||
}
|
||||
|
||||
func (self *cachedStorageDriver) Close() error {
|
||||
self.cache.Close()
|
||||
return self.backend.Close()
|
||||
}
|
||||
|
||||
// TODO(vishh): Cache all samples for a given duration and do not cap the maximum number of samples. This is useful if we happen to change the housekeeping duration.
|
||||
func MemoryCache(maxNumSamplesInCache, maxNumStatsInCache int, driver storage.StorageDriver) storage.StorageDriver {
|
||||
return &cachedStorageDriver{
|
||||
maxNumStatsInCache: maxNumStatsInCache,
|
||||
maxNumSamplesInCache: maxNumSamplesInCache,
|
||||
cache: memory.New(maxNumSamplesInCache, maxNumStatsInCache),
|
||||
backend: driver,
|
||||
}
|
||||
}
|
||||
108
storage/cache/memcache_test.go
vendored
108
storage/cache/memcache_test.go
vendored
@ -1,108 +0,0 @@
|
||||
// Copyright 2014 Google Inc. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cache
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/cadvisor/info"
|
||||
"github.com/google/cadvisor/storage"
|
||||
"github.com/google/cadvisor/storage/memory"
|
||||
"github.com/google/cadvisor/storage/test"
|
||||
)
|
||||
|
||||
type cacheTestStorageDriver struct {
|
||||
base storage.StorageDriver
|
||||
}
|
||||
|
||||
func (self *cacheTestStorageDriver) StatsEq(a, b *info.ContainerStats) bool {
|
||||
return test.DefaultStatsEq(a, b)
|
||||
}
|
||||
|
||||
func (self *cacheTestStorageDriver) AddStats(ref info.ContainerReference, stats *info.ContainerStats) error {
|
||||
return self.base.AddStats(ref, stats)
|
||||
}
|
||||
|
||||
func (self *cacheTestStorageDriver) RecentStats(containerName string, numStats int) ([]*info.ContainerStats, error) {
|
||||
return self.base.RecentStats(containerName, numStats)
|
||||
}
|
||||
|
||||
func (self *cacheTestStorageDriver) Percentiles(containerName string, cpuUsagePercentiles []int, memUsagePercentiles []int) (*info.ContainerStatsPercentiles, error) {
|
||||
return self.base.Percentiles(containerName, cpuUsagePercentiles, memUsagePercentiles)
|
||||
}
|
||||
|
||||
func (self *cacheTestStorageDriver) Samples(containerName string, numSamples int) ([]*info.ContainerStatsSample, error) {
|
||||
return self.base.Samples(containerName, numSamples)
|
||||
}
|
||||
|
||||
func (self *cacheTestStorageDriver) Close() error {
|
||||
return self.base.Close()
|
||||
}
|
||||
|
||||
func runStorageTest(f func(test.TestStorageDriver, *testing.T), t *testing.T) {
|
||||
maxSize := 200
|
||||
|
||||
for N := 10; N < maxSize; N += 10 {
|
||||
testDriver := &cacheTestStorageDriver{}
|
||||
backend := memory.New(N*2, N*2)
|
||||
testDriver.base = MemoryCache(N, N, backend)
|
||||
f(testDriver, t)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestMaxMemoryUsage(t *testing.T) {
|
||||
runStorageTest(test.StorageDriverTestMaxMemoryUsage, t)
|
||||
}
|
||||
|
||||
func TestSampleCpuUsage(t *testing.T) {
|
||||
runStorageTest(test.StorageDriverTestSampleCpuUsage, t)
|
||||
}
|
||||
|
||||
func TestSamplesWithoutSample(t *testing.T) {
|
||||
runStorageTest(test.StorageDriverTestSamplesWithoutSample, t)
|
||||
}
|
||||
|
||||
func TestPercentilesWithoutSample(t *testing.T) {
|
||||
runStorageTest(test.StorageDriverTestPercentilesWithoutSample, t)
|
||||
}
|
||||
|
||||
func TestPercentiles(t *testing.T) {
|
||||
N := 100
|
||||
testDriver := &cacheTestStorageDriver{}
|
||||
backend := memory.New(N*2, N*2)
|
||||
testDriver.base = MemoryCache(N, N, backend)
|
||||
test.StorageDriverTestPercentiles(testDriver, t)
|
||||
}
|
||||
|
||||
func TestRetrievePartialRecentStats(t *testing.T) {
|
||||
runStorageTest(test.StorageDriverTestRetrievePartialRecentStats, t)
|
||||
}
|
||||
|
||||
func TestRetrieveAllRecentStats(t *testing.T) {
|
||||
runStorageTest(test.StorageDriverTestRetrieveAllRecentStats, t)
|
||||
}
|
||||
|
||||
func TestNoRecentStats(t *testing.T) {
|
||||
runStorageTest(test.StorageDriverTestNoRecentStats, t)
|
||||
}
|
||||
|
||||
func TestNoSamples(t *testing.T) {
|
||||
runStorageTest(test.StorageDriverTestNoSamples, t)
|
||||
}
|
||||
|
||||
func TestPercentilesWithoutStats(t *testing.T) {
|
||||
runStorageTest(test.StorageDriverTestPercentilesWithoutStats, t)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user