From 4d0b365d432d9676984bd52f7019bf5ef1b70bea Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Mon, 7 Jul 2014 22:04:30 -0700 Subject: [PATCH 01/17] let users decide how many stats/samples they want to retrieve --- api/handler.go | 22 ++- cadvisor.go | 2 +- info/container.go | 28 ++++ info/test/datagen.go | 1 + manager/manager.go | 16 +- manager/manager_test.go | 337 ++++++++++++++++++++++++++++++++++++++++ pages/containers.go | 2 +- 7 files changed, 392 insertions(+), 16 deletions(-) create mode 100644 manager/manager_test.go diff --git a/api/handler.go b/api/handler.go index 409dc917..0740354a 100644 --- a/api/handler.go +++ b/api/handler.go @@ -21,10 +21,10 @@ import ( "fmt" "log" "net/http" - "net/url" "strings" "time" + "github.com/google/cadvisor/info" "github.com/google/cadvisor/manager" ) @@ -34,9 +34,11 @@ const ( MachineApi = "machine" ) -func HandleRequest(m manager.Manager, w http.ResponseWriter, u *url.URL) error { +func HandleRequest(m manager.Manager, w http.ResponseWriter, r *http.Request) error { start := time.Now() + u := r.URL + // Get API request type. requestType := u.Path[len(ApiResource):] i := strings.Index(requestType, "/") @@ -46,7 +48,8 @@ func HandleRequest(m manager.Manager, w http.ResponseWriter, u *url.URL) error { requestType = requestType[:i] } - if requestType == MachineApi { + switch { + case requestType == MachineApi: log.Printf("Api - Machine") // Get the MachineInfo @@ -60,14 +63,21 @@ func HandleRequest(m manager.Manager, w http.ResponseWriter, u *url.URL) error { fmt.Fprintf(w, "Failed to marshall MachineInfo with error: %s", err) } w.Write(out) - } else if requestType == ContainersApi { + case requestType == ContainersApi: // The container name is the path after the requestType containerName := requestArgs log.Printf("Api - Container(%s)", containerName) + var query info.ContainerInfoQuery + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(&query) + if err != nil { + fmt.Fprintf(w, "unable to decode the json value: %v", err) + return err + } // Get the container. - cont, err := m.GetContainerInfo(containerName) + cont, err := m.GetContainerInfo(containerName, &query) if err != nil { fmt.Fprintf(w, "Failed to get container \"%s\" with error: %s", containerName, err) return err @@ -79,7 +89,7 @@ func HandleRequest(m manager.Manager, w http.ResponseWriter, u *url.URL) error { fmt.Fprintf(w, "Failed to marshall container %q with error: %s", containerName, err) } w.Write(out) - } else { + default: return fmt.Errorf("unknown API request type %q", requestType) } diff --git a/cadvisor.go b/cadvisor.go index 8fe7a7a8..72c9e57b 100644 --- a/cadvisor.go +++ b/cadvisor.go @@ -79,7 +79,7 @@ func main() { // Handler for the API. http.HandleFunc(api.ApiResource, func(w http.ResponseWriter, r *http.Request) { - err := api.HandleRequest(containerManager, w, r.URL) + err := api.HandleRequest(containerManager, w, r) if err != nil { fmt.Fprintf(w, "%s", err) } diff --git a/info/container.go b/info/container.go index 6e987916..bf8a15f6 100644 --- a/info/container.go +++ b/info/container.go @@ -57,6 +57,34 @@ type ContainerReference struct { Aliases []string `json:"aliases,omitempty"` } +type ContainerInfoQuery struct { + NumStats int `json:"num_stats,omitempty"` + NumSamples int `json:"num_samples,omitempty"` + + CpuUsagePercentages []int `json:"cpu_usage_percentages,omitempty"` + MemoryUsagePercentages []int `json:"memory_usage_percentages,omitempty"` +} + +func (self *ContainerInfoQuery) FillWithDefaultValues() *ContainerInfoQuery { + ret := self + if ret == nil { + ret = new(ContainerInfoQuery) + } + if ret.NumStats <= 0 { + ret.NumStats = 1024 + } + if ret.NumSamples <= 0 { + ret.NumSamples = 1024 + } + if len(ret.CpuUsagePercentages) == 0 { + ret.CpuUsagePercentages = []int{50, 80, 90, 99} + } + if len(ret.MemoryUsagePercentages) == 0 { + ret.MemoryUsagePercentages = []int{50, 80, 90, 99} + } + return ret +} + type ContainerInfo struct { ContainerReference diff --git a/info/test/datagen.go b/info/test/datagen.go index 36841774..cb55ec45 100644 --- a/info/test/datagen.go +++ b/info/test/datagen.go @@ -46,6 +46,7 @@ func GenerateRandomStats(numStats, numCores int, duration time.Duration) []*info stats.Cpu.Usage.User = stats.Cpu.Usage.Total stats.Cpu.Usage.System = 0 stats.Memory.Usage = uint64(rand.Int63n(4096)) + ret[i] = stats } return ret } diff --git a/manager/manager.go b/manager/manager.go index 1b050355..07f09225 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -30,7 +30,7 @@ type Manager interface { Start() error // Get information about a container. - GetContainerInfo(containerName string) (*info.ContainerInfo, error) + GetContainerInfo(containerName string, query *info.ContainerInfoQuery) (*info.ContainerInfo, error) // Get information about the machine. GetMachineInfo() (*info.MachineInfo, error) @@ -106,8 +106,8 @@ func (m *manager) Start() error { } // Get a container by name. -func (m *manager) GetContainerInfo(containerName string) (*info.ContainerInfo, error) { - log.Printf("Get(%s)", containerName) +func (m *manager) GetContainerInfo(containerName string, query *info.ContainerInfoQuery) (*info.ContainerInfo, error) { + log.Printf("Get(%s); %+v", containerName, query) var cont *containerData var ok bool func() { @@ -130,21 +130,21 @@ func (m *manager) GetContainerInfo(containerName string) (*info.ContainerInfo, e var percentiles *info.ContainerStatsPercentiles var samples []*info.ContainerStatsSample var stats []*info.ContainerStats - // TODO(monnand): These numbers should not be hard coded + query = query.FillWithDefaultValues() percentiles, err = m.storageDriver.Percentiles( cinfo.Name, - []int{50, 80, 90, 99}, - []int{50, 80, 90, 99}, + query.CpuUsagePercentages, + query.MemoryUsagePercentages, ) if err != nil { return nil, err } - samples, err = m.storageDriver.Samples(cinfo.Name, 1024) + samples, err = m.storageDriver.Samples(cinfo.Name, query.NumSamples) if err != nil { return nil, err } - stats, err = m.storageDriver.RecentStats(cinfo.Name, 1024) + stats, err = m.storageDriver.RecentStats(cinfo.Name, query.NumStats) if err != nil { return nil, err } diff --git a/manager/manager_test.go b/manager/manager_test.go new file mode 100644 index 00000000..1d241d24 --- /dev/null +++ b/manager/manager_test.go @@ -0,0 +1,337 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Per-container manager. + +package manager + +import ( + "fmt" + "math/rand" + "reflect" + "testing" + "time" + + "github.com/google/cadvisor/container" + ctest "github.com/google/cadvisor/container/test" + "github.com/google/cadvisor/info" + itest "github.com/google/cadvisor/info/test" + stest "github.com/google/cadvisor/storage/test" +) + +func createManagerAndAddContainers( + driver *stest.MockStorageDriver, + containers []string, + f func(*ctest.MockContainerHandler), + t *testing.T, +) *manager { + if driver == nil { + driver = &stest.MockStorageDriver{} + } + factory := &ctest.FactoryForMockContainerHandler{ + Name: "factoryForManager", + PrepareContainerHandlerFunc: func(name string, handler *ctest.MockContainerHandler) { + handler.Name = name + found := false + for _, c := range containers { + if c == name { + found = true + } + } + if !found { + t.Errorf("Asked to create a container with name %v, which is unknown.", name) + } + f(handler) + }, + } + container.RegisterContainerHandlerFactory("/", factory) + mif, err := New(driver) + if err != nil { + t.Fatal(err) + } + if ret, ok := mif.(*manager); ok { + for _, container := range containers { + ret.containers[container], err = NewContainerData(container, driver) + if err != nil { + t.Fatal(err) + } + } + return ret + } + t.Fatal("Wrong type") + return nil +} + +func randomStatsAndSamples(numStats, numSamples int) ([]*info.ContainerStats, []*info.ContainerStatsSample, error) { + stats := itest.GenerateRandomStats(numStats, 4, 1*time.Second) + samples := make([]*info.ContainerStatsSample, 0, numSamples) + for i, s := range stats[1:] { + prev := stats[i] + sample, err := info.NewSample(prev, s) + if err != nil { + return nil, nil, fmt.Errorf("Unable to generate sample from %+v and %+v: %v", + prev, s, err) + } + samples = append(samples, sample) + if len(samples) == numSamples { + break + } + } + return stats, samples, nil +} + +func TestGetContainerInfo(t *testing.T) { + containers := []string{ + "/c1", + "/c2", + } + + query := &info.ContainerInfoQuery{ + NumStats: 256, + NumSamples: 128, + CpuUsagePercentages: []int{10, 50, 90}, + MemoryUsagePercentages: []int{10, 80, 90}, + } + + infosMap := make(map[string]*info.ContainerInfo, len(containers)) + handlerMap := make(map[string]*ctest.MockContainerHandler, len(containers)) + + for _, container := range containers { + stats, samples, err := randomStatsAndSamples(query.NumStats, query.NumSamples) + if err != nil { + t.Fatal(err) + } + + cpuPercentiles := make([]info.Percentile, 0, len(query.CpuUsagePercentages)) + for _, p := range query.CpuUsagePercentages { + percentile := info.Percentile{p, uint64(rand.Int63n(1000))} + cpuPercentiles = append(cpuPercentiles, percentile) + } + memPercentiles := make([]info.Percentile, 0, len(query.MemoryUsagePercentages)) + for _, p := range query.MemoryUsagePercentages { + percentile := info.Percentile{p, uint64(rand.Int63n(1000))} + memPercentiles = append(memPercentiles, percentile) + } + + percentiles := &info.ContainerStatsPercentiles{ + MaxMemoryUsage: uint64(rand.Int63n(4096)), + MemoryUsagePercentiles: memPercentiles, + CpuUsagePercentiles: cpuPercentiles, + } + + spec := itest.GenerateRandomContainerSpec(4) + + infosMap[container] = &info.ContainerInfo{ + ContainerReference: info.ContainerReference{ + Name: container, + }, + Spec: spec, + StatsPercentiles: percentiles, + Samples: samples, + Stats: stats, + } + } + + driver := &stest.MockStorageDriver{} + m := createManagerAndAddContainers( + driver, + containers, + func(h *ctest.MockContainerHandler) { + cinfo := infosMap[h.Name] + stats := cinfo.Stats + samples := cinfo.Samples + percentiles := cinfo.StatsPercentiles + spec := cinfo.Spec + driver.On( + "Percentiles", + h.Name, + query.CpuUsagePercentages, + query.MemoryUsagePercentages, + ).Return( + percentiles, + nil, + ) + + driver.On( + "Samples", + h.Name, + query.NumSamples, + ).Return( + samples, + nil, + ) + + driver.On( + "RecentStats", + h.Name, + query.NumStats, + ).Return( + stats, + nil, + ) + + h.On("ListContainers", container.LIST_SELF).Return( + []info.ContainerReference(nil), + nil, + ) + h.On("GetSpec").Return( + spec, + nil, + ) + handlerMap[h.Name] = h + }, + t, + ) + + returnedInfos := make(map[string]*info.ContainerInfo, len(containers)) + + for _, container := range containers { + cinfo, err := m.GetContainerInfo(container, query) + if err != nil { + t.Fatalf("Unable to get info for container %v: %v", container, err) + } + returnedInfos[container] = cinfo + } + + for container, handler := range handlerMap { + handler.AssertExpectations(t) + returned := returnedInfos[container] + expected := infosMap[container] + if !reflect.DeepEqual(returned, expected) { + t.Errorf("returned unexpected info for container %v; returned %+v; expected %+v", container, returned, expected) + } + } + +} + +func TestGetContainerInfoWithDefaultValue(t *testing.T) { + containers := []string{ + "/c1", + "/c2", + } + + var query *info.ContainerInfoQuery + query = query.FillWithDefaultValues() + + infosMap := make(map[string]*info.ContainerInfo, len(containers)) + handlerMap := make(map[string]*ctest.MockContainerHandler, len(containers)) + + for _, container := range containers { + stats, samples, err := randomStatsAndSamples(query.NumStats, query.NumSamples) + if err != nil { + t.Fatal(err) + } + + cpuPercentiles := make([]info.Percentile, 0, len(query.CpuUsagePercentages)) + for _, p := range query.CpuUsagePercentages { + percentile := info.Percentile{p, uint64(rand.Int63n(1000))} + cpuPercentiles = append(cpuPercentiles, percentile) + } + memPercentiles := make([]info.Percentile, 0, len(query.MemoryUsagePercentages)) + for _, p := range query.MemoryUsagePercentages { + percentile := info.Percentile{p, uint64(rand.Int63n(1000))} + memPercentiles = append(memPercentiles, percentile) + } + + percentiles := &info.ContainerStatsPercentiles{ + MaxMemoryUsage: uint64(rand.Int63n(4096)), + MemoryUsagePercentiles: memPercentiles, + CpuUsagePercentiles: cpuPercentiles, + } + + spec := itest.GenerateRandomContainerSpec(4) + + infosMap[container] = &info.ContainerInfo{ + ContainerReference: info.ContainerReference{ + Name: container, + }, + Spec: spec, + StatsPercentiles: percentiles, + Samples: samples, + Stats: stats, + } + } + + driver := &stest.MockStorageDriver{} + m := createManagerAndAddContainers( + driver, + containers, + func(h *ctest.MockContainerHandler) { + cinfo := infosMap[h.Name] + stats := cinfo.Stats + samples := cinfo.Samples + percentiles := cinfo.StatsPercentiles + spec := cinfo.Spec + driver.On( + "Percentiles", + h.Name, + query.CpuUsagePercentages, + query.MemoryUsagePercentages, + ).Return( + percentiles, + nil, + ) + + driver.On( + "Samples", + h.Name, + query.NumSamples, + ).Return( + samples, + nil, + ) + + driver.On( + "RecentStats", + h.Name, + query.NumStats, + ).Return( + stats, + nil, + ) + + h.On("ListContainers", container.LIST_SELF).Return( + []info.ContainerReference(nil), + nil, + ) + h.On("GetSpec").Return( + spec, + nil, + ) + handlerMap[h.Name] = h + }, + t, + ) + + returnedInfos := make(map[string]*info.ContainerInfo, len(containers)) + + for _, container := range containers { + // nil should give us default values + cinfo, err := m.GetContainerInfo(container, nil) + if err != nil { + t.Fatalf("Unable to get info for container %v: %v", container, err) + } + returnedInfos[container] = cinfo + } + + for container, handler := range handlerMap { + handler.AssertExpectations(t) + returned := returnedInfos[container] + expected := infosMap[container] + if !reflect.DeepEqual(returned, expected) { + t.Errorf("returned unexpected info for container %v; returned %+v; expected %+v", container, returned, expected) + } + } + +} diff --git a/pages/containers.go b/pages/containers.go index 58605a8b..d229775f 100644 --- a/pages/containers.go +++ b/pages/containers.go @@ -162,7 +162,7 @@ func ServerContainersPage(m manager.Manager, w http.ResponseWriter, u *url.URL) containerName := u.Path[len(ContainersPage)-1:] // Get the container. - cont, err := m.GetContainerInfo(containerName) + cont, err := m.GetContainerInfo(containerName, nil) if err != nil { return fmt.Errorf("Failed to get container \"%s\" with error: %s", containerName, err) } From 38c5f0e29f490f6ae053f9c6e56542710565e485 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Mon, 7 Jul 2014 22:10:25 -0700 Subject: [PATCH 02/17] empty query means default values --- api/handler.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/handler.go b/api/handler.go index 0740354a..104ee436 100644 --- a/api/handler.go +++ b/api/handler.go @@ -19,6 +19,7 @@ package api import ( "encoding/json" "fmt" + "io" "log" "net/http" "strings" @@ -72,8 +73,8 @@ func HandleRequest(m manager.Manager, w http.ResponseWriter, r *http.Request) er var query info.ContainerInfoQuery decoder := json.NewDecoder(r.Body) err := decoder.Decode(&query) - if err != nil { - fmt.Fprintf(w, "unable to decode the json value: %v", err) + if err != nil && err != io.EOF { + fmt.Fprintf(w, "unable to decode the json value: ", err) return err } // Get the container. From d932d351efc0b1c46033e6ab7dab240091bc2157 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 12:49:40 -0700 Subject: [PATCH 03/17] test on client library --- client/client.go | 25 +- client/client_test.go | 710 ++-------------------------------------- info/container.go | 17 + info/test/datagen.go | 40 +++ manager/manager_test.go | 88 +---- 5 files changed, 111 insertions(+), 769 deletions(-) diff --git a/client/client.go b/client/client.go index d521b1e3..452cb5f8 100644 --- a/client/client.go +++ b/client/client.go @@ -15,6 +15,7 @@ package cadvisor import ( + "bytes" "encoding/json" "fmt" "io/ioutil" @@ -49,7 +50,7 @@ func (self *Client) machineInfoUrl() string { func (self *Client) MachineInfo() (minfo *info.MachineInfo, err error) { u := self.machineInfoUrl() ret := new(info.MachineInfo) - err = self.httpGetJsonData(ret, u, "machine info") + err = self.httpGetJsonData(ret, nil, u, "machine info") if err != nil { return } @@ -64,8 +65,19 @@ func (self *Client) containerInfoUrl(name string) string { return strings.Join([]string{self.baseUrl, "containers", name}, "/") } -func (self *Client) httpGetJsonData(data interface{}, url, infoName string) error { - resp, err := http.Get(url) +func (self *Client) httpGetJsonData(data, postData interface{}, url, infoName string) error { + var resp *http.Response + var err error + + if postData != nil { + data, err := json.Marshal(postData) + if err != nil { + return fmt.Errorf("unable to marshal data: %v", err) + } + resp, err = http.Post(url, "application/json", bytes.NewBuffer(data)) + } else { + resp, err = http.Get(url) + } if err != nil { err = fmt.Errorf("unable to get %v: %v", infoName, err) return err @@ -84,10 +96,13 @@ func (self *Client) httpGetJsonData(data interface{}, url, infoName string) erro return nil } -func (self *Client) ContainerInfo(name string) (cinfo *info.ContainerInfo, err error) { +func (self *Client) ContainerInfo( + name string, + query *info.ContainerInfoQuery, +) (cinfo *info.ContainerInfo, err error) { u := self.containerInfoUrl(name) ret := new(info.ContainerInfo) - err = self.httpGetJsonData(ret, u, fmt.Sprintf("container info for %v", name)) + err = self.httpGetJsonData(ret, query, u, fmt.Sprintf("container info for %v", name)) if err != nil { return } diff --git a/client/client_test.go b/client/client_test.go index 6fc8eaad..28e04db9 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -21,33 +21,39 @@ import ( "net/http/httptest" "reflect" "testing" + "time" "github.com/google/cadvisor/info" + itest "github.com/google/cadvisor/info/test" ) func testGetJsonData( - strRep string, - emptyData interface{}, + expected interface{}, f func() (interface{}, error), ) error { - err := json.Unmarshal([]byte(strRep), emptyData) - if err != nil { - return fmt.Errorf("invalid json input: %v", err) - } reply, err := f() if err != nil { return fmt.Errorf("unable to retrieve data: %v", err) } - if !reflect.DeepEqual(reply, emptyData) { - return fmt.Errorf("retrieved wrong data: %+v != %+v", reply, emptyData) + if !reflect.DeepEqual(reply, expected) { + return fmt.Errorf("retrieved wrong data: %+v != %+v", reply, expected) } return nil } -func cadvisorTestClient(path, reply string) (*Client, *httptest.Server, error) { +func cadvisorTestClient(path string, expectedPostObj, expectedPostObjEmpty, replyObj interface{}) (*Client, *httptest.Server, error) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == path { - fmt.Fprint(w, reply) + if expectedPostObj != nil { + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(expectedPostObjEmpty) + if err != nil { + fmt.Fprintf(w, "received invalid json object: %v", err) + return + } + } + encoder := json.NewEncoder(w) + encoder.Encode(replyObj) } else if r.URL.Path == "/api/v1.0/machine" { fmt.Fprint(w, `{"num_cores":8,"memory_capacity":31625871360}`) } else { @@ -64,13 +70,16 @@ func cadvisorTestClient(path, reply string) (*Client, *httptest.Server, error) { } func TestGetMachineinfo(t *testing.T) { - respStr := `{"num_cores":8,"memory_capacity":31625871360}` - client, server, err := cadvisorTestClient("/api/v1.0/machine", respStr) + minfo := &info.MachineInfo{ + NumCores: 8, + MemoryCapacity: 31625871360, + } + client, server, err := cadvisorTestClient("/api/v1.0/machine", nil, nil, minfo) if err != nil { t.Fatalf("unable to get a client %v", err) } defer server.Close() - err = testGetJsonData(respStr, &info.MachineInfo{}, func() (interface{}, error) { + err = testGetJsonData(minfo, func() (interface{}, error) { return client.MachineInfo() }) if err != nil { @@ -79,676 +88,21 @@ func TestGetMachineinfo(t *testing.T) { } func TestGetContainerInfo(t *testing.T) { - respStr := ` -{ - "name": "%v", - "spec": { - "cpu": { - "limit": 18446744073709551000, - "max_limit": 0, - "mask": { - "data": [ - 18446744073709551000 - ] - } - }, - "memory": { - "limit": 18446744073709551000, - "swap_limit": 18446744073709551000 - } - }, - "stats": [ - { - "timestamp": "2014-06-13T01:03:26.434981825Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:27.538394608Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:28.640302072Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:29.74247308Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:30.844494537Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:31.946757066Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:33.050214062Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:34.15222186Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:35.25417391Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:36.355902169Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:37.457585928Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:38.559417379Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:39.662978029Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:40.764671232Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:41.866456459Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - } - ], - "stats_summary": { - "max_memory_usage": 495616, - "samples": [ - { - "timestamp": "2014-06-13T01:03:27.538394608Z", - "duration": 1103412783, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:28.640302072Z", - "duration": 1101907464, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:29.74247308Z", - "duration": 1102171008, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:30.844494537Z", - "duration": 1102021457, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:31.946757066Z", - "duration": 1102262529, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:33.050214062Z", - "duration": 1103456996, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:34.15222186Z", - "duration": 1102007798, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:35.25417391Z", - "duration": 1101952050, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:36.355902169Z", - "duration": 1101728259, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:37.457585928Z", - "duration": 1101683759, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:38.559417379Z", - "duration": 1101831451, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:39.662978029Z", - "duration": 1103560650, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:40.764671232Z", - "duration": 1101693203, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:41.866456459Z", - "duration": 1101785227, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - } - ], - "memory_usage_percentiles": [ - { - "percentage": 50, - "value": 495616 - }, - { - "percentage": 80, - "value": 495616 - }, - { - "percentage": 90, - "value": 495616 - }, - { - "percentage": 95, - "value": 495616 - }, - { - "percentage": 99, - "value": 495616 - } - ], - "cpu_usage_percentiles": [ - { - "percentage": 50, - "value": 0 - }, - { - "percentage": 80, - "value": 0 - }, - { - "percentage": 90, - "value": 0 - }, - { - "percentage": 95, - "value": 0 - }, - { - "percentage": 99, - "value": 0 - } - ] - } -} -` + query := &info.ContainerInfoQuery{ + NumStats: 512, + NumSamples: 256, + CpuUsagePercentages: []int{10, 50, 90}, + MemoryUsagePercentages: []int{10, 80, 90}, + } containerName := "/some/container" - respStr = fmt.Sprintf(respStr, containerName) - client, server, err := cadvisorTestClient(fmt.Sprintf("/api/v1.0/containers%v", containerName), respStr) + cinfo := itest.GenerateRandomContainerInfo(containerName, 4, query, 1*time.Second) + client, server, err := cadvisorTestClient(fmt.Sprintf("/api/v1.0/containers%v", containerName), query, &info.ContainerInfoQuery{}, cinfo) if err != nil { t.Fatalf("unable to get a client %v", err) } defer server.Close() - err = testGetJsonData(respStr, &info.ContainerInfo{}, func() (interface{}, error) { - return client.ContainerInfo(containerName) + err = testGetJsonData(cinfo, func() (interface{}, error) { + return client.ContainerInfo(containerName, nil) }) if err != nil { t.Fatal(err) diff --git a/info/container.go b/info/container.go index bf8a15f6..702fe316 100644 --- a/info/container.go +++ b/info/container.go @@ -308,6 +308,23 @@ func NewSample(prev, current *ContainerStats) (*ContainerStatsSample, error) { return sample, nil } +func NewSamplesFromStats(stats ...*ContainerStats) ([]*ContainerStatsSample, error) { + if len(stats) < 2 { + return nil, nil + } + samples := make([]*ContainerStatsSample, 0, len(stats)-1) + for i, s := range stats[1:] { + prev := stats[i] + sample, err := NewSample(prev, s) + if err != nil { + return nil, fmt.Errorf("Unable to generate sample from %+v and %+v: %v", + prev, s, err) + } + samples = append(samples, sample) + } + return samples, nil +} + type uint64Slice []uint64 func (self uint64Slice) Len() int { diff --git a/info/test/datagen.go b/info/test/datagen.go index cb55ec45..3f6872bc 100644 --- a/info/test/datagen.go +++ b/info/test/datagen.go @@ -67,3 +67,43 @@ func GenerateRandomContainerSpec(numCores int) *info.ContainerSpec { ret.Memory.Limit = uint64(4096 + rand.Int63n(4096)) return ret } + +func GenerateRandomContainerInfo(containerName string, numCores int, query *info.ContainerInfoQuery, duration time.Duration) *info.ContainerInfo { + stats := GenerateRandomStats(query.NumStats, numCores, duration) + samples, _ := info.NewSamplesFromStats(stats...) + if len(samples) > query.NumSamples { + samples = samples[:query.NumSamples] + } + cpuPercentiles := make([]info.Percentile, 0, len(query.CpuUsagePercentages)) + + // TODO(monnand): This will generate percentiles where 50%tile data may + // be larger than 90%tile data. + for _, p := range query.CpuUsagePercentages { + percentile := info.Percentile{p, uint64(rand.Int63n(1000))} + cpuPercentiles = append(cpuPercentiles, percentile) + } + memPercentiles := make([]info.Percentile, 0, len(query.MemoryUsagePercentages)) + for _, p := range query.MemoryUsagePercentages { + percentile := info.Percentile{p, uint64(rand.Int63n(1000))} + memPercentiles = append(memPercentiles, percentile) + } + + percentiles := &info.ContainerStatsPercentiles{ + MaxMemoryUsage: uint64(rand.Int63n(4096)), + MemoryUsagePercentiles: memPercentiles, + CpuUsagePercentiles: cpuPercentiles, + } + + spec := GenerateRandomContainerSpec(numCores) + + ret := &info.ContainerInfo{ + ContainerReference: info.ContainerReference{ + Name: containerName, + }, + Spec: spec, + StatsPercentiles: percentiles, + Samples: samples, + Stats: stats, + } + return ret +} diff --git a/manager/manager_test.go b/manager/manager_test.go index 1d241d24..48a12c5e 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -17,8 +17,6 @@ package manager import ( - "fmt" - "math/rand" "reflect" "testing" "time" @@ -73,24 +71,6 @@ func createManagerAndAddContainers( return nil } -func randomStatsAndSamples(numStats, numSamples int) ([]*info.ContainerStats, []*info.ContainerStatsSample, error) { - stats := itest.GenerateRandomStats(numStats, 4, 1*time.Second) - samples := make([]*info.ContainerStatsSample, 0, numSamples) - for i, s := range stats[1:] { - prev := stats[i] - sample, err := info.NewSample(prev, s) - if err != nil { - return nil, nil, fmt.Errorf("Unable to generate sample from %+v and %+v: %v", - prev, s, err) - } - samples = append(samples, sample) - if len(samples) == numSamples { - break - } - } - return stats, samples, nil -} - func TestGetContainerInfo(t *testing.T) { containers := []string{ "/c1", @@ -108,39 +88,7 @@ func TestGetContainerInfo(t *testing.T) { handlerMap := make(map[string]*ctest.MockContainerHandler, len(containers)) for _, container := range containers { - stats, samples, err := randomStatsAndSamples(query.NumStats, query.NumSamples) - if err != nil { - t.Fatal(err) - } - - cpuPercentiles := make([]info.Percentile, 0, len(query.CpuUsagePercentages)) - for _, p := range query.CpuUsagePercentages { - percentile := info.Percentile{p, uint64(rand.Int63n(1000))} - cpuPercentiles = append(cpuPercentiles, percentile) - } - memPercentiles := make([]info.Percentile, 0, len(query.MemoryUsagePercentages)) - for _, p := range query.MemoryUsagePercentages { - percentile := info.Percentile{p, uint64(rand.Int63n(1000))} - memPercentiles = append(memPercentiles, percentile) - } - - percentiles := &info.ContainerStatsPercentiles{ - MaxMemoryUsage: uint64(rand.Int63n(4096)), - MemoryUsagePercentiles: memPercentiles, - CpuUsagePercentiles: cpuPercentiles, - } - - spec := itest.GenerateRandomContainerSpec(4) - - infosMap[container] = &info.ContainerInfo{ - ContainerReference: info.ContainerReference{ - Name: container, - }, - Spec: spec, - StatsPercentiles: percentiles, - Samples: samples, - Stats: stats, - } + infosMap[container] = itest.GenerateRandomContainerInfo(container, 4, query, 1*time.Second) } driver := &stest.MockStorageDriver{} @@ -228,39 +176,7 @@ func TestGetContainerInfoWithDefaultValue(t *testing.T) { handlerMap := make(map[string]*ctest.MockContainerHandler, len(containers)) for _, container := range containers { - stats, samples, err := randomStatsAndSamples(query.NumStats, query.NumSamples) - if err != nil { - t.Fatal(err) - } - - cpuPercentiles := make([]info.Percentile, 0, len(query.CpuUsagePercentages)) - for _, p := range query.CpuUsagePercentages { - percentile := info.Percentile{p, uint64(rand.Int63n(1000))} - cpuPercentiles = append(cpuPercentiles, percentile) - } - memPercentiles := make([]info.Percentile, 0, len(query.MemoryUsagePercentages)) - for _, p := range query.MemoryUsagePercentages { - percentile := info.Percentile{p, uint64(rand.Int63n(1000))} - memPercentiles = append(memPercentiles, percentile) - } - - percentiles := &info.ContainerStatsPercentiles{ - MaxMemoryUsage: uint64(rand.Int63n(4096)), - MemoryUsagePercentiles: memPercentiles, - CpuUsagePercentiles: cpuPercentiles, - } - - spec := itest.GenerateRandomContainerSpec(4) - - infosMap[container] = &info.ContainerInfo{ - ContainerReference: info.ContainerReference{ - Name: container, - }, - Spec: spec, - StatsPercentiles: percentiles, - Samples: samples, - Stats: stats, - } + infosMap[container] = itest.GenerateRandomContainerInfo(container, 4, query, 1*time.Second) } driver := &stest.MockStorageDriver{} From d6c5b6a64e55a32d15d8fd721daeca45be342975 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 13:01:20 -0700 Subject: [PATCH 04/17] client test --- client/client_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 28e04db9..61e1bc3e 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -41,15 +41,17 @@ func testGetJsonData( return nil } -func cadvisorTestClient(path string, expectedPostObj, expectedPostObjEmpty, replyObj interface{}) (*Client, *httptest.Server, error) { +func cadvisorTestClient(path string, expectedPostObj, expectedPostObjEmpty, replyObj interface{}, t *testing.T) (*Client, *httptest.Server, error) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == path { if expectedPostObj != nil { decoder := json.NewDecoder(r.Body) err := decoder.Decode(expectedPostObjEmpty) if err != nil { - fmt.Fprintf(w, "received invalid json object: %v", err) - return + t.Errorf("Recived invalid object: %v", err) + } + if !reflect.DeepEqual(expectedPostObj, expectedPostObjEmpty) { + t.Errorf("Recived unexpected object: %+v", expectedPostObjEmpty) } } encoder := json.NewEncoder(w) @@ -74,7 +76,7 @@ func TestGetMachineinfo(t *testing.T) { NumCores: 8, MemoryCapacity: 31625871360, } - client, server, err := cadvisorTestClient("/api/v1.0/machine", nil, nil, minfo) + client, server, err := cadvisorTestClient("/api/v1.0/machine", nil, nil, minfo, t) if err != nil { t.Fatalf("unable to get a client %v", err) } @@ -96,13 +98,13 @@ func TestGetContainerInfo(t *testing.T) { } containerName := "/some/container" cinfo := itest.GenerateRandomContainerInfo(containerName, 4, query, 1*time.Second) - client, server, err := cadvisorTestClient(fmt.Sprintf("/api/v1.0/containers%v", containerName), query, &info.ContainerInfoQuery{}, cinfo) + client, server, err := cadvisorTestClient(fmt.Sprintf("/api/v1.0/containers%v", containerName), query, &info.ContainerInfoQuery{}, cinfo, t) if err != nil { t.Fatalf("unable to get a client %v", err) } defer server.Close() err = testGetJsonData(cinfo, func() (interface{}, error) { - return client.ContainerInfo(containerName, nil) + return client.ContainerInfo(containerName, query) }) if err != nil { t.Fatal(err) From 18f531b5fb183b7c0c5ef158096cacc9f017aa6d Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 13:12:29 -0700 Subject: [PATCH 05/17] smaller data set --- client/client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 61e1bc3e..b5db5870 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -91,8 +91,8 @@ func TestGetMachineinfo(t *testing.T) { func TestGetContainerInfo(t *testing.T) { query := &info.ContainerInfoQuery{ - NumStats: 512, - NumSamples: 256, + NumStats: 3, + NumSamples: 2, CpuUsagePercentages: []int{10, 50, 90}, MemoryUsagePercentages: []int{10, 80, 90}, } From a3d9f2d09488b923a528e8af11de995d19e97aa4 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 13:22:30 -0700 Subject: [PATCH 06/17] better error message for unit test --- client/client_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/client_test.go b/client/client_test.go index b5db5870..b61773ec 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -25,6 +25,7 @@ import ( "github.com/google/cadvisor/info" itest "github.com/google/cadvisor/info/test" + "github.com/kr/pretty" ) func testGetJsonData( @@ -36,7 +37,7 @@ func testGetJsonData( return fmt.Errorf("unable to retrieve data: %v", err) } if !reflect.DeepEqual(reply, expected) { - return fmt.Errorf("retrieved wrong data: %+v != %+v", reply, expected) + return pretty.Errorf("retrieved wrong data: %# v != %# v", reply, expected) } return nil } From 6efeaf6b8795011f02421f8641d79bf8f77882c7 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 13:35:05 -0700 Subject: [PATCH 07/17] Heisenbug? --- client/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/client_test.go b/client/client_test.go index b61773ec..0e3d7cc7 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -37,7 +37,7 @@ func testGetJsonData( return fmt.Errorf("unable to retrieve data: %v", err) } if !reflect.DeepEqual(reply, expected) { - return pretty.Errorf("retrieved wrong data: %# v != %# v", reply, expected) + return fmt.Errorf("retrieved wrong data: %v != %v", pretty.Sprintf("%# v", reply), pretty.Sprintf("% #v", expected)) } return nil } From 115b132e5f12538676258c96ed335b062ac26985 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 13:55:09 -0700 Subject: [PATCH 08/17] stats time precesion affects the test results --- client/client_test.go | 42 ++++++++++++++++++++++++----- info/container.go | 62 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 7 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 0e3d7cc7..62c0f8e4 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -37,7 +37,7 @@ func testGetJsonData( return fmt.Errorf("unable to retrieve data: %v", err) } if !reflect.DeepEqual(reply, expected) { - return fmt.Errorf("retrieved wrong data: %v != %v", pretty.Sprintf("%# v", reply), pretty.Sprintf("% #v", expected)) + return pretty.Errorf("retrieved wrong data: %# v != %# v", reply, expected) } return nil } @@ -82,12 +82,13 @@ func TestGetMachineinfo(t *testing.T) { t.Fatalf("unable to get a client %v", err) } defer server.Close() - err = testGetJsonData(minfo, func() (interface{}, error) { - return client.MachineInfo() - }) + returned, err := client.MachineInfo() if err != nil { t.Fatal(err) } + if !reflect.DeepEqual(returned, minfo) { + t.Fatalf("received unexpected machine info") + } } func TestGetContainerInfo(t *testing.T) { @@ -104,10 +105,37 @@ func TestGetContainerInfo(t *testing.T) { t.Fatalf("unable to get a client %v", err) } defer server.Close() - err = testGetJsonData(cinfo, func() (interface{}, error) { - return client.ContainerInfo(containerName, query) - }) + returned, err := client.ContainerInfo(containerName, query) if err != nil { t.Fatal(err) } + + // We cannot use DeepEqual() to compare them directly, + // because json en/decoded time may have preceision issues. + if !reflect.DeepEqual(returned.ContainerReference, cinfo.ContainerReference) { + t.Errorf("received unexpected container ref") + } + if !reflect.DeepEqual(returned.Subcontainers, cinfo.Subcontainers) { + t.Errorf("received unexpected subcontainers") + } + if !reflect.DeepEqual(returned.Spec, cinfo.Spec) { + t.Errorf("received unexpected spec") + } + if !reflect.DeepEqual(returned.StatsPercentiles, cinfo.StatsPercentiles) { + t.Errorf("received unexpected spec") + } + + for i, expectedStats := range cinfo.Stats { + returnedStats := returned.Stats[i] + if !expectedStats.Eq(returnedStats) { + t.Errorf("received unexpected stats") + } + } + + for i, expectedSample := range cinfo.Samples { + returnedSample := returned.Samples[i] + if !expectedSample.Eq(returnedSample) { + t.Errorf("received unexpected sample") + } + } } diff --git a/info/container.go b/info/container.go index 702fe316..55410a79 100644 --- a/info/container.go +++ b/info/container.go @@ -16,6 +16,7 @@ package info import ( "fmt" + "reflect" "sort" "time" ) @@ -247,6 +248,67 @@ type ContainerStatsSample struct { } `json:"memory"` } +func timeEq(t1, t2 time.Time, tolerance time.Duration) bool { + // t1 should not be later than t2 + if t1.After(t2) { + t1, t2 = t2, t1 + } + diff := t2.Sub(t1) + if diff <= tolerance { + return true + } + return false +} + +func durationEq(a, b time.Duration, tolerance time.Duration) bool { + if a > b { + a, b = b, a + } + diff := a - b + if diff <= tolerance { + return true + } + return false +} + +const ( + // 10ms, i.e. 0.01s + timePrecision time.Duration = 10 * time.Millisecond +) + +// This function is useful because we do not require precise time +// representation. +func (a *ContainerStats) Eq(b *ContainerStats) bool { + if !timeEq(a.Timestamp, b.Timestamp, timePrecision) { + return false + } + if !reflect.DeepEqual(a.Cpu, b.Cpu) { + return false + } + if !reflect.DeepEqual(a.Memory, b.Memory) { + return false + } + return true +} + +// This function is useful because we do not require precise time +// representation. +func (a *ContainerStatsSample) Eq(b *ContainerStatsSample) bool { + if !timeEq(a.Timestamp, b.Timestamp, timePrecision) { + return false + } + if !durationEq(a.Duration, b.Duration, timePrecision) { + return false + } + if !reflect.DeepEqual(a.Cpu, b.Cpu) { + return false + } + if !reflect.DeepEqual(a.Memory, b.Memory) { + return false + } + return true +} + type Percentile struct { Percentage int `json:"percentage"` Value uint64 `json:"value"` From f2a0365766b8f14eadad9d5749e14520a9bf9730 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 17:20:04 -0700 Subject: [PATCH 09/17] gofmt -r "CpuUsagePercentages->CpuUsagePercentiles" --- client/client_test.go | 2 +- info/container.go | 9 +++++---- info/test/datagen.go | 4 ++-- manager/manager.go | 2 +- manager/manager_test.go | 6 +++--- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 62c0f8e4..32890971 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -95,7 +95,7 @@ func TestGetContainerInfo(t *testing.T) { query := &info.ContainerInfoQuery{ NumStats: 3, NumSamples: 2, - CpuUsagePercentages: []int{10, 50, 90}, + CpuUsagePercentiles: []int{10, 50, 90}, MemoryUsagePercentages: []int{10, 80, 90}, } containerName := "/some/container" diff --git a/info/container.go b/info/container.go index 55410a79..ea3ea642 100644 --- a/info/container.go +++ b/info/container.go @@ -58,12 +58,13 @@ type ContainerReference struct { Aliases []string `json:"aliases,omitempty"` } +// ContainerInfoQuery specifies how much data users want to get about a container type ContainerInfoQuery struct { NumStats int `json:"num_stats,omitempty"` NumSamples int `json:"num_samples,omitempty"` - CpuUsagePercentages []int `json:"cpu_usage_percentages,omitempty"` - MemoryUsagePercentages []int `json:"memory_usage_percentages,omitempty"` + CpuUsagePercentiles []int `json:"cpu_usage_percentiles,omitempty"` + MemoryUsagePercentages []int `json:"memory_usage_percentiles,omitempty"` } func (self *ContainerInfoQuery) FillWithDefaultValues() *ContainerInfoQuery { @@ -77,8 +78,8 @@ func (self *ContainerInfoQuery) FillWithDefaultValues() *ContainerInfoQuery { if ret.NumSamples <= 0 { ret.NumSamples = 1024 } - if len(ret.CpuUsagePercentages) == 0 { - ret.CpuUsagePercentages = []int{50, 80, 90, 99} + if len(ret.CpuUsagePercentiles) == 0 { + ret.CpuUsagePercentiles = []int{50, 80, 90, 99} } if len(ret.MemoryUsagePercentages) == 0 { ret.MemoryUsagePercentages = []int{50, 80, 90, 99} diff --git a/info/test/datagen.go b/info/test/datagen.go index 3f6872bc..48c160b3 100644 --- a/info/test/datagen.go +++ b/info/test/datagen.go @@ -74,11 +74,11 @@ func GenerateRandomContainerInfo(containerName string, numCores int, query *info if len(samples) > query.NumSamples { samples = samples[:query.NumSamples] } - cpuPercentiles := make([]info.Percentile, 0, len(query.CpuUsagePercentages)) + cpuPercentiles := make([]info.Percentile, 0, len(query.CpuUsagePercentiles)) // TODO(monnand): This will generate percentiles where 50%tile data may // be larger than 90%tile data. - for _, p := range query.CpuUsagePercentages { + for _, p := range query.CpuUsagePercentiles { percentile := info.Percentile{p, uint64(rand.Int63n(1000))} cpuPercentiles = append(cpuPercentiles, percentile) } diff --git a/manager/manager.go b/manager/manager.go index 07f09225..8ed243ce 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -133,7 +133,7 @@ func (m *manager) GetContainerInfo(containerName string, query *info.ContainerIn query = query.FillWithDefaultValues() percentiles, err = m.storageDriver.Percentiles( cinfo.Name, - query.CpuUsagePercentages, + query.CpuUsagePercentiles, query.MemoryUsagePercentages, ) if err != nil { diff --git a/manager/manager_test.go b/manager/manager_test.go index 48a12c5e..851acce2 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -80,7 +80,7 @@ func TestGetContainerInfo(t *testing.T) { query := &info.ContainerInfoQuery{ NumStats: 256, NumSamples: 128, - CpuUsagePercentages: []int{10, 50, 90}, + CpuUsagePercentiles: []int{10, 50, 90}, MemoryUsagePercentages: []int{10, 80, 90}, } @@ -104,7 +104,7 @@ func TestGetContainerInfo(t *testing.T) { driver.On( "Percentiles", h.Name, - query.CpuUsagePercentages, + query.CpuUsagePercentiles, query.MemoryUsagePercentages, ).Return( percentiles, @@ -192,7 +192,7 @@ func TestGetContainerInfoWithDefaultValue(t *testing.T) { driver.On( "Percentiles", h.Name, - query.CpuUsagePercentages, + query.CpuUsagePercentiles, query.MemoryUsagePercentages, ).Return( percentiles, From d8e9f8e5a96407df1ed5e67087d3427549d57df2 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 17:20:47 -0700 Subject: [PATCH 10/17] gofmt -r "FillWithDefaultValues->FillDefaults" --- info/container.go | 2 +- manager/manager.go | 2 +- manager/manager_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/info/container.go b/info/container.go index ea3ea642..7b839752 100644 --- a/info/container.go +++ b/info/container.go @@ -67,7 +67,7 @@ type ContainerInfoQuery struct { MemoryUsagePercentages []int `json:"memory_usage_percentiles,omitempty"` } -func (self *ContainerInfoQuery) FillWithDefaultValues() *ContainerInfoQuery { +func (self *ContainerInfoQuery) FillDefaults() *ContainerInfoQuery { ret := self if ret == nil { ret = new(ContainerInfoQuery) diff --git a/manager/manager.go b/manager/manager.go index 8ed243ce..8a7352f6 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -130,7 +130,7 @@ func (m *manager) GetContainerInfo(containerName string, query *info.ContainerIn var percentiles *info.ContainerStatsPercentiles var samples []*info.ContainerStatsSample var stats []*info.ContainerStats - query = query.FillWithDefaultValues() + query = query.FillDefaults() percentiles, err = m.storageDriver.Percentiles( cinfo.Name, query.CpuUsagePercentiles, diff --git a/manager/manager_test.go b/manager/manager_test.go index 851acce2..68cb95ba 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -170,7 +170,7 @@ func TestGetContainerInfoWithDefaultValue(t *testing.T) { } var query *info.ContainerInfoQuery - query = query.FillWithDefaultValues() + query = query.FillDefaults() infosMap := make(map[string]*info.ContainerInfo, len(containers)) handlerMap := make(map[string]*ctest.MockContainerHandler, len(containers)) From d8acc132402a400f7aa3c5b38ae0f9b4e7a19e0b Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 17:21:46 -0700 Subject: [PATCH 11/17] docs --- info/container.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/info/container.go b/info/container.go index 7b839752..1e3f06aa 100644 --- a/info/container.go +++ b/info/container.go @@ -58,7 +58,8 @@ type ContainerReference struct { Aliases []string `json:"aliases,omitempty"` } -// ContainerInfoQuery specifies how much data users want to get about a container +// ContainerInfoQuery is used when users check a container info from the REST api. +// It specifies how much data users want to get about a container type ContainerInfoQuery struct { NumStats int `json:"num_stats,omitempty"` NumSamples int `json:"num_samples,omitempty"` From c1ecbc94a61a014118b99889bd49ee61dc8d82bb Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 17:30:35 -0700 Subject: [PATCH 12/17] fmt.Errorf() --- api/handler.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/handler.go b/api/handler.go index 104ee436..8bf5e433 100644 --- a/api/handler.go +++ b/api/handler.go @@ -74,8 +74,7 @@ func HandleRequest(m manager.Manager, w http.ResponseWriter, r *http.Request) er decoder := json.NewDecoder(r.Body) err := decoder.Decode(&query) if err != nil && err != io.EOF { - fmt.Fprintf(w, "unable to decode the json value: ", err) - return err + return fmt.Errorf("unable to decode the json value: ", err) } // Get the container. cont, err := m.GetContainerInfo(containerName, &query) From 6a5a395fbd7a1e46dc92a1113fed749e38e7dd20 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 18:04:13 -0700 Subject: [PATCH 13/17] fix typo --- client/client_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 32890971..56c8452f 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -49,10 +49,10 @@ func cadvisorTestClient(path string, expectedPostObj, expectedPostObjEmpty, repl decoder := json.NewDecoder(r.Body) err := decoder.Decode(expectedPostObjEmpty) if err != nil { - t.Errorf("Recived invalid object: %v", err) + t.Errorf("Recieved invalid object: %v", err) } if !reflect.DeepEqual(expectedPostObj, expectedPostObjEmpty) { - t.Errorf("Recived unexpected object: %+v", expectedPostObjEmpty) + t.Errorf("Recieved unexpected object: %+v", expectedPostObjEmpty) } } encoder := json.NewEncoder(w) @@ -111,7 +111,7 @@ func TestGetContainerInfo(t *testing.T) { } // We cannot use DeepEqual() to compare them directly, - // because json en/decoded time may have preceision issues. + // because json en/decoded time may have precision issues. if !reflect.DeepEqual(returned.ContainerReference, cinfo.ContainerReference) { t.Errorf("received unexpected container ref") } From 6818ac9b9f873fd037f7bcc75c6eada7d03c8406 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 18:04:57 -0700 Subject: [PATCH 14/17] gofmt -r "ContainerInfoQuery->ContainerInfoRequest" --- api/handler.go | 2 +- client/client.go | 3 +-- client/client_test.go | 4 ++-- info/container.go | 6 +++--- info/test/datagen.go | 2 +- manager/manager.go | 4 ++-- manager/manager_test.go | 4 ++-- 7 files changed, 12 insertions(+), 13 deletions(-) diff --git a/api/handler.go b/api/handler.go index 8bf5e433..393705bd 100644 --- a/api/handler.go +++ b/api/handler.go @@ -70,7 +70,7 @@ func HandleRequest(m manager.Manager, w http.ResponseWriter, r *http.Request) er log.Printf("Api - Container(%s)", containerName) - var query info.ContainerInfoQuery + var query info.ContainerInfoRequest decoder := json.NewDecoder(r.Body) err := decoder.Decode(&query) if err != nil && err != io.EOF { diff --git a/client/client.go b/client/client.go index 452cb5f8..033cbc2a 100644 --- a/client/client.go +++ b/client/client.go @@ -98,8 +98,7 @@ func (self *Client) httpGetJsonData(data, postData interface{}, url, infoName st func (self *Client) ContainerInfo( name string, - query *info.ContainerInfoQuery, -) (cinfo *info.ContainerInfo, err error) { + query *info.ContainerInfoRequest) (cinfo *info.ContainerInfo, err error) { u := self.containerInfoUrl(name) ret := new(info.ContainerInfo) err = self.httpGetJsonData(ret, query, u, fmt.Sprintf("container info for %v", name)) diff --git a/client/client_test.go b/client/client_test.go index 56c8452f..a5b7f691 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -92,7 +92,7 @@ func TestGetMachineinfo(t *testing.T) { } func TestGetContainerInfo(t *testing.T) { - query := &info.ContainerInfoQuery{ + query := &info.ContainerInfoRequest{ NumStats: 3, NumSamples: 2, CpuUsagePercentiles: []int{10, 50, 90}, @@ -100,7 +100,7 @@ func TestGetContainerInfo(t *testing.T) { } containerName := "/some/container" cinfo := itest.GenerateRandomContainerInfo(containerName, 4, query, 1*time.Second) - client, server, err := cadvisorTestClient(fmt.Sprintf("/api/v1.0/containers%v", containerName), query, &info.ContainerInfoQuery{}, cinfo, t) + client, server, err := cadvisorTestClient(fmt.Sprintf("/api/v1.0/containers%v", containerName), query, &info.ContainerInfoRequest{}, cinfo, t) if err != nil { t.Fatalf("unable to get a client %v", err) } diff --git a/info/container.go b/info/container.go index 1e3f06aa..8c79ea29 100644 --- a/info/container.go +++ b/info/container.go @@ -60,7 +60,7 @@ type ContainerReference struct { // ContainerInfoQuery is used when users check a container info from the REST api. // It specifies how much data users want to get about a container -type ContainerInfoQuery struct { +type ContainerInfoRequest struct { NumStats int `json:"num_stats,omitempty"` NumSamples int `json:"num_samples,omitempty"` @@ -68,10 +68,10 @@ type ContainerInfoQuery struct { MemoryUsagePercentages []int `json:"memory_usage_percentiles,omitempty"` } -func (self *ContainerInfoQuery) FillDefaults() *ContainerInfoQuery { +func (self *ContainerInfoRequest) FillDefaults() *ContainerInfoRequest { ret := self if ret == nil { - ret = new(ContainerInfoQuery) + ret = new(ContainerInfoRequest) } if ret.NumStats <= 0 { ret.NumStats = 1024 diff --git a/info/test/datagen.go b/info/test/datagen.go index 48c160b3..f130ad5e 100644 --- a/info/test/datagen.go +++ b/info/test/datagen.go @@ -68,7 +68,7 @@ func GenerateRandomContainerSpec(numCores int) *info.ContainerSpec { return ret } -func GenerateRandomContainerInfo(containerName string, numCores int, query *info.ContainerInfoQuery, duration time.Duration) *info.ContainerInfo { +func GenerateRandomContainerInfo(containerName string, numCores int, query *info.ContainerInfoRequest, duration time.Duration) *info.ContainerInfo { stats := GenerateRandomStats(query.NumStats, numCores, duration) samples, _ := info.NewSamplesFromStats(stats...) if len(samples) > query.NumSamples { diff --git a/manager/manager.go b/manager/manager.go index 8a7352f6..696e0bad 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -30,7 +30,7 @@ type Manager interface { Start() error // Get information about a container. - GetContainerInfo(containerName string, query *info.ContainerInfoQuery) (*info.ContainerInfo, error) + GetContainerInfo(containerName string, query *info.ContainerInfoRequest) (*info.ContainerInfo, error) // Get information about the machine. GetMachineInfo() (*info.MachineInfo, error) @@ -106,7 +106,7 @@ func (m *manager) Start() error { } // Get a container by name. -func (m *manager) GetContainerInfo(containerName string, query *info.ContainerInfoQuery) (*info.ContainerInfo, error) { +func (m *manager) GetContainerInfo(containerName string, query *info.ContainerInfoRequest) (*info.ContainerInfo, error) { log.Printf("Get(%s); %+v", containerName, query) var cont *containerData var ok bool diff --git a/manager/manager_test.go b/manager/manager_test.go index 68cb95ba..99862c5d 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -77,7 +77,7 @@ func TestGetContainerInfo(t *testing.T) { "/c2", } - query := &info.ContainerInfoQuery{ + query := &info.ContainerInfoRequest{ NumStats: 256, NumSamples: 128, CpuUsagePercentiles: []int{10, 50, 90}, @@ -169,7 +169,7 @@ func TestGetContainerInfoWithDefaultValue(t *testing.T) { "/c2", } - var query *info.ContainerInfoQuery + var query *info.ContainerInfoRequest query = query.FillDefaults() infosMap := make(map[string]*info.ContainerInfo, len(containers)) From 942236addeec01fb3964569379bfad86affcad35 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 18:11:35 -0700 Subject: [PATCH 15/17] docs --- info/container.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/info/container.go b/info/container.go index 8c79ea29..20a603d2 100644 --- a/info/container.go +++ b/info/container.go @@ -61,10 +61,14 @@ type ContainerReference struct { // ContainerInfoQuery is used when users check a container info from the REST api. // It specifies how much data users want to get about a container type ContainerInfoRequest struct { - NumStats int `json:"num_stats,omitempty"` + // Max number of stats to return. + NumStats int `json:"num_stats,omitempty"` + // Max number of samples to return. NumSamples int `json:"num_samples,omitempty"` - CpuUsagePercentiles []int `json:"cpu_usage_percentiles,omitempty"` + // Different percentiles of CPU usage within a period. The values must be within [0, 100] + CpuUsagePercentiles []int `json:"cpu_usage_percentiles,omitempty"` + // Different percentiles of memory usage within a period. The values must be within [0, 100] MemoryUsagePercentages []int `json:"memory_usage_percentiles,omitempty"` } From c7f69c2382234d085d684f2022f6507736a24916 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Thu, 10 Jul 2014 10:41:20 -0700 Subject: [PATCH 16/17] move function into test --- info/container.go | 2 ++ info/test/datagen.go | 20 +++++++++++++++++++- pages/containers.go | 6 +++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/info/container.go b/info/container.go index 20a603d2..2d3ed9b0 100644 --- a/info/container.go +++ b/info/container.go @@ -376,6 +376,7 @@ func NewSample(prev, current *ContainerStats) (*ContainerStatsSample, error) { return sample, nil } +/* func NewSamplesFromStats(stats ...*ContainerStats) ([]*ContainerStatsSample, error) { if len(stats) < 2 { return nil, nil @@ -392,6 +393,7 @@ func NewSamplesFromStats(stats ...*ContainerStats) ([]*ContainerStatsSample, err } return samples, nil } +*/ type uint64Slice []uint64 diff --git a/info/test/datagen.go b/info/test/datagen.go index f130ad5e..a1c0a356 100644 --- a/info/test/datagen.go +++ b/info/test/datagen.go @@ -15,6 +15,7 @@ package test import ( + "fmt" "math" "math/rand" "time" @@ -70,7 +71,7 @@ func GenerateRandomContainerSpec(numCores int) *info.ContainerSpec { func GenerateRandomContainerInfo(containerName string, numCores int, query *info.ContainerInfoRequest, duration time.Duration) *info.ContainerInfo { stats := GenerateRandomStats(query.NumStats, numCores, duration) - samples, _ := info.NewSamplesFromStats(stats...) + samples, _ := NewSamplesFromStats(stats...) if len(samples) > query.NumSamples { samples = samples[:query.NumSamples] } @@ -107,3 +108,20 @@ func GenerateRandomContainerInfo(containerName string, numCores int, query *info } return ret } + +func NewSamplesFromStats(stats ...*info.ContainerStats) ([]*info.ContainerStatsSample, error) { + if len(stats) < 2 { + return nil, nil + } + samples := make([]*info.ContainerStatsSample, 0, len(stats)-1) + for i, s := range stats[1:] { + prev := stats[i] + sample, err := info.NewSample(prev, s) + if err != nil { + return nil, fmt.Errorf("Unable to generate sample from %+v and %+v: %v", + prev, s, err) + } + samples = append(samples, sample) + } + return samples, nil +} diff --git a/pages/containers.go b/pages/containers.go index d229775f..b3302b17 100644 --- a/pages/containers.go +++ b/pages/containers.go @@ -162,7 +162,11 @@ func ServerContainersPage(m manager.Manager, w http.ResponseWriter, u *url.URL) containerName := u.Path[len(ContainersPage)-1:] // Get the container. - cont, err := m.GetContainerInfo(containerName, nil) + reqParams := info.ContainerInfoRequest{ + NumStats: 60, + NumSamples: 60, + } + cont, err := m.GetContainerInfo(containerName, &reqParams) if err != nil { return fmt.Errorf("Failed to get container \"%s\" with error: %s", containerName, err) } From d5279b54b28aea42ef87357dc57353549c250079 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Thu, 10 Jul 2014 11:21:37 -0700 Subject: [PATCH 17/17] remove some commented out code --- info/container.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/info/container.go b/info/container.go index 2d3ed9b0..cd6a7175 100644 --- a/info/container.go +++ b/info/container.go @@ -376,25 +376,6 @@ func NewSample(prev, current *ContainerStats) (*ContainerStatsSample, error) { return sample, nil } -/* -func NewSamplesFromStats(stats ...*ContainerStats) ([]*ContainerStatsSample, error) { - if len(stats) < 2 { - return nil, nil - } - samples := make([]*ContainerStatsSample, 0, len(stats)-1) - for i, s := range stats[1:] { - prev := stats[i] - sample, err := NewSample(prev, s) - if err != nil { - return nil, fmt.Errorf("Unable to generate sample from %+v and %+v: %v", - prev, s, err) - } - samples = append(samples, sample) - } - return samples, nil -} -*/ - type uint64Slice []uint64 func (self uint64Slice) Len() int {