From d932d351efc0b1c46033e6ab7dab240091bc2157 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 8 Jul 2014 12:49:40 -0700 Subject: [PATCH] test on client library --- client/client.go | 25 +- client/client_test.go | 710 ++-------------------------------------- info/container.go | 17 + info/test/datagen.go | 40 +++ manager/manager_test.go | 88 +---- 5 files changed, 111 insertions(+), 769 deletions(-) diff --git a/client/client.go b/client/client.go index d521b1e3..452cb5f8 100644 --- a/client/client.go +++ b/client/client.go @@ -15,6 +15,7 @@ package cadvisor import ( + "bytes" "encoding/json" "fmt" "io/ioutil" @@ -49,7 +50,7 @@ func (self *Client) machineInfoUrl() string { func (self *Client) MachineInfo() (minfo *info.MachineInfo, err error) { u := self.machineInfoUrl() ret := new(info.MachineInfo) - err = self.httpGetJsonData(ret, u, "machine info") + err = self.httpGetJsonData(ret, nil, u, "machine info") if err != nil { return } @@ -64,8 +65,19 @@ func (self *Client) containerInfoUrl(name string) string { return strings.Join([]string{self.baseUrl, "containers", name}, "/") } -func (self *Client) httpGetJsonData(data interface{}, url, infoName string) error { - resp, err := http.Get(url) +func (self *Client) httpGetJsonData(data, postData interface{}, url, infoName string) error { + var resp *http.Response + var err error + + if postData != nil { + data, err := json.Marshal(postData) + if err != nil { + return fmt.Errorf("unable to marshal data: %v", err) + } + resp, err = http.Post(url, "application/json", bytes.NewBuffer(data)) + } else { + resp, err = http.Get(url) + } if err != nil { err = fmt.Errorf("unable to get %v: %v", infoName, err) return err @@ -84,10 +96,13 @@ func (self *Client) httpGetJsonData(data interface{}, url, infoName string) erro return nil } -func (self *Client) ContainerInfo(name string) (cinfo *info.ContainerInfo, err error) { +func (self *Client) ContainerInfo( + name string, + query *info.ContainerInfoQuery, +) (cinfo *info.ContainerInfo, err error) { u := self.containerInfoUrl(name) ret := new(info.ContainerInfo) - err = self.httpGetJsonData(ret, u, fmt.Sprintf("container info for %v", name)) + err = self.httpGetJsonData(ret, query, u, fmt.Sprintf("container info for %v", name)) if err != nil { return } diff --git a/client/client_test.go b/client/client_test.go index 6fc8eaad..28e04db9 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -21,33 +21,39 @@ import ( "net/http/httptest" "reflect" "testing" + "time" "github.com/google/cadvisor/info" + itest "github.com/google/cadvisor/info/test" ) func testGetJsonData( - strRep string, - emptyData interface{}, + expected interface{}, f func() (interface{}, error), ) error { - err := json.Unmarshal([]byte(strRep), emptyData) - if err != nil { - return fmt.Errorf("invalid json input: %v", err) - } reply, err := f() if err != nil { return fmt.Errorf("unable to retrieve data: %v", err) } - if !reflect.DeepEqual(reply, emptyData) { - return fmt.Errorf("retrieved wrong data: %+v != %+v", reply, emptyData) + if !reflect.DeepEqual(reply, expected) { + return fmt.Errorf("retrieved wrong data: %+v != %+v", reply, expected) } return nil } -func cadvisorTestClient(path, reply string) (*Client, *httptest.Server, error) { +func cadvisorTestClient(path string, expectedPostObj, expectedPostObjEmpty, replyObj interface{}) (*Client, *httptest.Server, error) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == path { - fmt.Fprint(w, reply) + if expectedPostObj != nil { + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(expectedPostObjEmpty) + if err != nil { + fmt.Fprintf(w, "received invalid json object: %v", err) + return + } + } + encoder := json.NewEncoder(w) + encoder.Encode(replyObj) } else if r.URL.Path == "/api/v1.0/machine" { fmt.Fprint(w, `{"num_cores":8,"memory_capacity":31625871360}`) } else { @@ -64,13 +70,16 @@ func cadvisorTestClient(path, reply string) (*Client, *httptest.Server, error) { } func TestGetMachineinfo(t *testing.T) { - respStr := `{"num_cores":8,"memory_capacity":31625871360}` - client, server, err := cadvisorTestClient("/api/v1.0/machine", respStr) + minfo := &info.MachineInfo{ + NumCores: 8, + MemoryCapacity: 31625871360, + } + client, server, err := cadvisorTestClient("/api/v1.0/machine", nil, nil, minfo) if err != nil { t.Fatalf("unable to get a client %v", err) } defer server.Close() - err = testGetJsonData(respStr, &info.MachineInfo{}, func() (interface{}, error) { + err = testGetJsonData(minfo, func() (interface{}, error) { return client.MachineInfo() }) if err != nil { @@ -79,676 +88,21 @@ func TestGetMachineinfo(t *testing.T) { } func TestGetContainerInfo(t *testing.T) { - respStr := ` -{ - "name": "%v", - "spec": { - "cpu": { - "limit": 18446744073709551000, - "max_limit": 0, - "mask": { - "data": [ - 18446744073709551000 - ] - } - }, - "memory": { - "limit": 18446744073709551000, - "swap_limit": 18446744073709551000 - } - }, - "stats": [ - { - "timestamp": "2014-06-13T01:03:26.434981825Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:27.538394608Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:28.640302072Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:29.74247308Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:30.844494537Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:31.946757066Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:33.050214062Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:34.15222186Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:35.25417391Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:36.355902169Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:37.457585928Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:38.559417379Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:39.662978029Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:40.764671232Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - }, - { - "timestamp": "2014-06-13T01:03:41.866456459Z", - "cpu": { - "usage": { - "total": 56896502, - "per_cpu_usage": [ - 20479682, - 13579420, - 6025040, - 2255123, - 3635661, - 2489368, - 5158288, - 3273920 - ], - "user": 10000000, - "system": 30000000 - }, - "load": 0 - }, - "memory": { - "usage": 495616, - "container_data": { - "pgfault": 2279 - }, - "hierarchical_data": { - "pgfault": 2279 - } - } - } - ], - "stats_summary": { - "max_memory_usage": 495616, - "samples": [ - { - "timestamp": "2014-06-13T01:03:27.538394608Z", - "duration": 1103412783, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:28.640302072Z", - "duration": 1101907464, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:29.74247308Z", - "duration": 1102171008, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:30.844494537Z", - "duration": 1102021457, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:31.946757066Z", - "duration": 1102262529, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:33.050214062Z", - "duration": 1103456996, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:34.15222186Z", - "duration": 1102007798, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:35.25417391Z", - "duration": 1101952050, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:36.355902169Z", - "duration": 1101728259, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:37.457585928Z", - "duration": 1101683759, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:38.559417379Z", - "duration": 1101831451, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:39.662978029Z", - "duration": 1103560650, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:40.764671232Z", - "duration": 1101693203, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - }, - { - "timestamp": "2014-06-13T01:03:41.866456459Z", - "duration": 1101785227, - "cpu": { - "usage": 0 - }, - "memory": { - "usage": 495616 - } - } - ], - "memory_usage_percentiles": [ - { - "percentage": 50, - "value": 495616 - }, - { - "percentage": 80, - "value": 495616 - }, - { - "percentage": 90, - "value": 495616 - }, - { - "percentage": 95, - "value": 495616 - }, - { - "percentage": 99, - "value": 495616 - } - ], - "cpu_usage_percentiles": [ - { - "percentage": 50, - "value": 0 - }, - { - "percentage": 80, - "value": 0 - }, - { - "percentage": 90, - "value": 0 - }, - { - "percentage": 95, - "value": 0 - }, - { - "percentage": 99, - "value": 0 - } - ] - } -} -` + query := &info.ContainerInfoQuery{ + NumStats: 512, + NumSamples: 256, + CpuUsagePercentages: []int{10, 50, 90}, + MemoryUsagePercentages: []int{10, 80, 90}, + } containerName := "/some/container" - respStr = fmt.Sprintf(respStr, containerName) - client, server, err := cadvisorTestClient(fmt.Sprintf("/api/v1.0/containers%v", containerName), respStr) + cinfo := itest.GenerateRandomContainerInfo(containerName, 4, query, 1*time.Second) + client, server, err := cadvisorTestClient(fmt.Sprintf("/api/v1.0/containers%v", containerName), query, &info.ContainerInfoQuery{}, cinfo) if err != nil { t.Fatalf("unable to get a client %v", err) } defer server.Close() - err = testGetJsonData(respStr, &info.ContainerInfo{}, func() (interface{}, error) { - return client.ContainerInfo(containerName) + err = testGetJsonData(cinfo, func() (interface{}, error) { + return client.ContainerInfo(containerName, nil) }) if err != nil { t.Fatal(err) diff --git a/info/container.go b/info/container.go index bf8a15f6..702fe316 100644 --- a/info/container.go +++ b/info/container.go @@ -308,6 +308,23 @@ func NewSample(prev, current *ContainerStats) (*ContainerStatsSample, error) { return sample, nil } +func NewSamplesFromStats(stats ...*ContainerStats) ([]*ContainerStatsSample, error) { + if len(stats) < 2 { + return nil, nil + } + samples := make([]*ContainerStatsSample, 0, len(stats)-1) + for i, s := range stats[1:] { + prev := stats[i] + sample, err := NewSample(prev, s) + if err != nil { + return nil, fmt.Errorf("Unable to generate sample from %+v and %+v: %v", + prev, s, err) + } + samples = append(samples, sample) + } + return samples, nil +} + type uint64Slice []uint64 func (self uint64Slice) Len() int { diff --git a/info/test/datagen.go b/info/test/datagen.go index cb55ec45..3f6872bc 100644 --- a/info/test/datagen.go +++ b/info/test/datagen.go @@ -67,3 +67,43 @@ func GenerateRandomContainerSpec(numCores int) *info.ContainerSpec { ret.Memory.Limit = uint64(4096 + rand.Int63n(4096)) return ret } + +func GenerateRandomContainerInfo(containerName string, numCores int, query *info.ContainerInfoQuery, duration time.Duration) *info.ContainerInfo { + stats := GenerateRandomStats(query.NumStats, numCores, duration) + samples, _ := info.NewSamplesFromStats(stats...) + if len(samples) > query.NumSamples { + samples = samples[:query.NumSamples] + } + cpuPercentiles := make([]info.Percentile, 0, len(query.CpuUsagePercentages)) + + // TODO(monnand): This will generate percentiles where 50%tile data may + // be larger than 90%tile data. + for _, p := range query.CpuUsagePercentages { + percentile := info.Percentile{p, uint64(rand.Int63n(1000))} + cpuPercentiles = append(cpuPercentiles, percentile) + } + memPercentiles := make([]info.Percentile, 0, len(query.MemoryUsagePercentages)) + for _, p := range query.MemoryUsagePercentages { + percentile := info.Percentile{p, uint64(rand.Int63n(1000))} + memPercentiles = append(memPercentiles, percentile) + } + + percentiles := &info.ContainerStatsPercentiles{ + MaxMemoryUsage: uint64(rand.Int63n(4096)), + MemoryUsagePercentiles: memPercentiles, + CpuUsagePercentiles: cpuPercentiles, + } + + spec := GenerateRandomContainerSpec(numCores) + + ret := &info.ContainerInfo{ + ContainerReference: info.ContainerReference{ + Name: containerName, + }, + Spec: spec, + StatsPercentiles: percentiles, + Samples: samples, + Stats: stats, + } + return ret +} diff --git a/manager/manager_test.go b/manager/manager_test.go index 1d241d24..48a12c5e 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -17,8 +17,6 @@ package manager import ( - "fmt" - "math/rand" "reflect" "testing" "time" @@ -73,24 +71,6 @@ func createManagerAndAddContainers( return nil } -func randomStatsAndSamples(numStats, numSamples int) ([]*info.ContainerStats, []*info.ContainerStatsSample, error) { - stats := itest.GenerateRandomStats(numStats, 4, 1*time.Second) - samples := make([]*info.ContainerStatsSample, 0, numSamples) - for i, s := range stats[1:] { - prev := stats[i] - sample, err := info.NewSample(prev, s) - if err != nil { - return nil, nil, fmt.Errorf("Unable to generate sample from %+v and %+v: %v", - prev, s, err) - } - samples = append(samples, sample) - if len(samples) == numSamples { - break - } - } - return stats, samples, nil -} - func TestGetContainerInfo(t *testing.T) { containers := []string{ "/c1", @@ -108,39 +88,7 @@ func TestGetContainerInfo(t *testing.T) { handlerMap := make(map[string]*ctest.MockContainerHandler, len(containers)) for _, container := range containers { - stats, samples, err := randomStatsAndSamples(query.NumStats, query.NumSamples) - if err != nil { - t.Fatal(err) - } - - cpuPercentiles := make([]info.Percentile, 0, len(query.CpuUsagePercentages)) - for _, p := range query.CpuUsagePercentages { - percentile := info.Percentile{p, uint64(rand.Int63n(1000))} - cpuPercentiles = append(cpuPercentiles, percentile) - } - memPercentiles := make([]info.Percentile, 0, len(query.MemoryUsagePercentages)) - for _, p := range query.MemoryUsagePercentages { - percentile := info.Percentile{p, uint64(rand.Int63n(1000))} - memPercentiles = append(memPercentiles, percentile) - } - - percentiles := &info.ContainerStatsPercentiles{ - MaxMemoryUsage: uint64(rand.Int63n(4096)), - MemoryUsagePercentiles: memPercentiles, - CpuUsagePercentiles: cpuPercentiles, - } - - spec := itest.GenerateRandomContainerSpec(4) - - infosMap[container] = &info.ContainerInfo{ - ContainerReference: info.ContainerReference{ - Name: container, - }, - Spec: spec, - StatsPercentiles: percentiles, - Samples: samples, - Stats: stats, - } + infosMap[container] = itest.GenerateRandomContainerInfo(container, 4, query, 1*time.Second) } driver := &stest.MockStorageDriver{} @@ -228,39 +176,7 @@ func TestGetContainerInfoWithDefaultValue(t *testing.T) { handlerMap := make(map[string]*ctest.MockContainerHandler, len(containers)) for _, container := range containers { - stats, samples, err := randomStatsAndSamples(query.NumStats, query.NumSamples) - if err != nil { - t.Fatal(err) - } - - cpuPercentiles := make([]info.Percentile, 0, len(query.CpuUsagePercentages)) - for _, p := range query.CpuUsagePercentages { - percentile := info.Percentile{p, uint64(rand.Int63n(1000))} - cpuPercentiles = append(cpuPercentiles, percentile) - } - memPercentiles := make([]info.Percentile, 0, len(query.MemoryUsagePercentages)) - for _, p := range query.MemoryUsagePercentages { - percentile := info.Percentile{p, uint64(rand.Int63n(1000))} - memPercentiles = append(memPercentiles, percentile) - } - - percentiles := &info.ContainerStatsPercentiles{ - MaxMemoryUsage: uint64(rand.Int63n(4096)), - MemoryUsagePercentiles: memPercentiles, - CpuUsagePercentiles: cpuPercentiles, - } - - spec := itest.GenerateRandomContainerSpec(4) - - infosMap[container] = &info.ContainerInfo{ - ContainerReference: info.ContainerReference{ - Name: container, - }, - Spec: spec, - StatsPercentiles: percentiles, - Samples: samples, - Stats: stats, - } + infosMap[container] = itest.GenerateRandomContainerInfo(container, 4, query, 1*time.Second) } driver := &stest.MockStorageDriver{}