From 62b37eca1d99188b2ee51d6114510070380729f7 Mon Sep 17 00:00:00 2001 From: Victor Marmol Date: Tue, 14 Oct 2014 03:54:17 -0700 Subject: [PATCH] Adding --log_cadvisor_usage flag to log the resource usage of cAdvisor. This helps during performance analysis. --- Godeps/Godeps.json | 10 ++ .../docker/docker/pkg/units/MAINTAINERS | 2 + .../docker/docker/pkg/units/duration.go | 31 ++++++ .../docker/docker/pkg/units/duration_test.go | 52 ++++++++++ .../docker/docker/pkg/units/size.go | 94 +++++++++++++++++++ .../docker/docker/pkg/units/size_test.go | 89 ++++++++++++++++++ manager/container.go | 46 +++++++-- manager/container_test.go | 2 +- manager/manager.go | 35 ++++--- manager/manager_test.go | 2 +- 10 files changed, 342 insertions(+), 21 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/units/MAINTAINERS create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/units/duration.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/units/duration_test.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/units/size.go create mode 100644 Godeps/_workspace/src/github.com/docker/docker/pkg/units/size_test.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 0d813926..99a92c1e 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -5,6 +5,11 @@ "./..." ], "Deps": [ + { + "ImportPath": "code.google.com/p/go.exp/inotify", + "Comment": "null-75", + "Rev": "bd8df7009305d6ada223ea3c95b94c0f38bfa119" + }, { "ImportPath": "code.google.com/p/goauth2/oauth", "Comment": "weekly-56", @@ -43,6 +48,11 @@ "Comment": "v1.1.1-402-g487a417", "Rev": "487a417d9fd074d0e78876072c7d1ebfd398ea7a" }, + { + "ImportPath": "github.com/docker/docker/pkg/units", + "Comment": "v1.1.1-402-g487a417", + "Rev": "487a417d9fd074d0e78876072c7d1ebfd398ea7a" + }, { "ImportPath": "github.com/docker/libcontainer", "Comment": "v1.2.0-46-g0da391f", diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/units/MAINTAINERS b/Godeps/_workspace/src/github.com/docker/docker/pkg/units/MAINTAINERS new file mode 100644 index 00000000..68a97d2f --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/units/MAINTAINERS @@ -0,0 +1,2 @@ +Michael Crosby (@crosbymichael) +Victor Vieux (@vieux) diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/units/duration.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/units/duration.go new file mode 100644 index 00000000..cd331214 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/units/duration.go @@ -0,0 +1,31 @@ +package units + +import ( + "fmt" + "time" +) + +// HumanDuration returns a human-readable approximation of a duration +// (eg. "About a minute", "4 hours ago", etc.) +func HumanDuration(d time.Duration) string { + if seconds := int(d.Seconds()); seconds < 1 { + return "Less than a second" + } else if seconds < 60 { + return fmt.Sprintf("%d seconds", seconds) + } else if minutes := int(d.Minutes()); minutes == 1 { + return "About a minute" + } else if minutes < 60 { + return fmt.Sprintf("%d minutes", minutes) + } else if hours := int(d.Hours()); hours == 1 { + return "About an hour" + } else if hours < 48 { + return fmt.Sprintf("%d hours", hours) + } else if hours < 24*7*2 { + return fmt.Sprintf("%d days", hours/24) + } else if hours < 24*30*3 { + return fmt.Sprintf("%d weeks", hours/24/7) + } else if hours < 24*365*2 { + return fmt.Sprintf("%d months", hours/24/30) + } + return fmt.Sprintf("%f years", d.Hours()/24/365) +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/units/duration_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/units/duration_test.go new file mode 100644 index 00000000..4906f8a1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/units/duration_test.go @@ -0,0 +1,52 @@ +package units + +import ( + "testing" + "time" +) + +func TestHumanDuration(t *testing.T) { + // Useful duration abstractions + day := 24 * time.Hour + week := 7 * day + month := 30 * day + year := 365 * day + + assertEquals(t, "Less than a second", HumanDuration(450*time.Millisecond)) + assertEquals(t, "47 seconds", HumanDuration(47*time.Second)) + assertEquals(t, "About a minute", HumanDuration(1*time.Minute)) + assertEquals(t, "3 minutes", HumanDuration(3*time.Minute)) + assertEquals(t, "35 minutes", HumanDuration(35*time.Minute)) + assertEquals(t, "35 minutes", HumanDuration(35*time.Minute+40*time.Second)) + assertEquals(t, "About an hour", HumanDuration(1*time.Hour)) + assertEquals(t, "About an hour", HumanDuration(1*time.Hour+45*time.Minute)) + assertEquals(t, "3 hours", HumanDuration(3*time.Hour)) + assertEquals(t, "3 hours", HumanDuration(3*time.Hour+59*time.Minute)) + assertEquals(t, "4 hours", HumanDuration(3*time.Hour+60*time.Minute)) + assertEquals(t, "24 hours", HumanDuration(24*time.Hour)) + assertEquals(t, "36 hours", HumanDuration(1*day+12*time.Hour)) + assertEquals(t, "2 days", HumanDuration(2*day)) + assertEquals(t, "7 days", HumanDuration(7*day)) + assertEquals(t, "13 days", HumanDuration(13*day+5*time.Hour)) + assertEquals(t, "2 weeks", HumanDuration(2*week)) + assertEquals(t, "2 weeks", HumanDuration(2*week+4*day)) + assertEquals(t, "3 weeks", HumanDuration(3*week)) + assertEquals(t, "4 weeks", HumanDuration(4*week)) + assertEquals(t, "4 weeks", HumanDuration(4*week+3*day)) + assertEquals(t, "4 weeks", HumanDuration(1*month)) + assertEquals(t, "6 weeks", HumanDuration(1*month+2*week)) + assertEquals(t, "8 weeks", HumanDuration(2*month)) + assertEquals(t, "3 months", HumanDuration(3*month+1*week)) + assertEquals(t, "5 months", HumanDuration(5*month+2*week)) + assertEquals(t, "13 months", HumanDuration(13*month)) + assertEquals(t, "23 months", HumanDuration(23*month)) + assertEquals(t, "24 months", HumanDuration(24*month)) + assertEquals(t, "2.010959 years", HumanDuration(24*month+2*week)) + assertEquals(t, "3.164384 years", HumanDuration(3*year+2*month)) +} + +func assertEquals(t *testing.T, expected, actual interface{}) { + if expected != actual { + t.Errorf("Expected '%s' but got '%s'", expected, actual) + } +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/units/size.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/units/size.go new file mode 100644 index 00000000..e6e9f21b --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/units/size.go @@ -0,0 +1,94 @@ +package units + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// HumanSize returns a human-readable approximation of a size +// using SI standard (eg. "44kB", "17MB") +func HumanSize(size int64) string { + i := 0 + var sizef float64 + sizef = float64(size) + units := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"} + for sizef >= 1000.0 { + sizef = sizef / 1000.0 + i++ + } + return fmt.Sprintf("%.4g %s", sizef, units[i]) +} + +// FromHumanSize returns an integer from a human-readable specification of a size +// using SI standard (eg. "44kB", "17MB") +func FromHumanSize(size string) (int64, error) { + re, err := regexp.Compile("^(\\d+)([kKmMgGtTpP])?[bB]?$") + if err != nil { + return -1, fmt.Errorf("%s does not specify not a size", size) + } + + matches := re.FindStringSubmatch(size) + + if len(matches) != 3 { + return -1, fmt.Errorf("Invalid size: '%s'", size) + } + + theSize, err := strconv.ParseInt(matches[1], 10, 0) + if err != nil { + return -1, err + } + + unit := strings.ToLower(matches[2]) + + if unit == "k" { + theSize *= 1000 + } else if unit == "m" { + theSize *= 1000 * 1000 + } else if unit == "g" { + theSize *= 1000 * 1000 * 1000 + } else if unit == "t" { + theSize *= 1000 * 1000 * 1000 * 1000 + } else if unit == "p" { + theSize *= 1000 * 1000 * 1000 * 1000 * 1000 + } + + return theSize, nil +} + +// Parses a human-readable string representing an amount of RAM +// in bytes, kibibytes, mebibytes, gibibytes, or tebibytes and +// returns the number of bytes, or -1 if the string is unparseable. +// Units are case-insensitive, and the 'b' suffix is optional. +func RAMInBytes(size string) (int64, error) { + re, err := regexp.Compile("^(\\d+)([kKmMgGtT])?[bB]?$") + if err != nil { + return -1, err + } + + matches := re.FindStringSubmatch(size) + + if len(matches) != 3 { + return -1, fmt.Errorf("Invalid size: '%s'", size) + } + + memLimit, err := strconv.ParseInt(matches[1], 10, 0) + if err != nil { + return -1, err + } + + unit := strings.ToLower(matches[2]) + + if unit == "k" { + memLimit *= 1024 + } else if unit == "m" { + memLimit *= 1024 * 1024 + } else if unit == "g" { + memLimit *= 1024 * 1024 * 1024 + } else if unit == "t" { + memLimit *= 1024 * 1024 * 1024 * 1024 + } + + return memLimit, nil +} diff --git a/Godeps/_workspace/src/github.com/docker/docker/pkg/units/size_test.go b/Godeps/_workspace/src/github.com/docker/docker/pkg/units/size_test.go new file mode 100644 index 00000000..de8b44f8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/docker/pkg/units/size_test.go @@ -0,0 +1,89 @@ +package units + +import ( + "testing" +) + +func TestHumanSize(t *testing.T) { + assertEquals(t, "1 kB", HumanSize(1000)) + assertEquals(t, "1.024 kB", HumanSize(1024)) + assertEquals(t, "1 MB", HumanSize(1000000)) + assertEquals(t, "1.049 MB", HumanSize(1048576)) + assertEquals(t, "2 MB", HumanSize(2*1000*1000)) + assertEquals(t, "3.42 GB", HumanSize(3.42*1000*1000*1000)) + assertEquals(t, "5.372 TB", HumanSize(5.372*1000*1000*1000*1000)) + assertEquals(t, "2.22 PB", HumanSize(2.22*1000*1000*1000*1000*1000)) + assertEquals(t, "2.22 EB", HumanSize(2.22*1000*1000*1000*1000*1000*1000)) + assertEquals(t, "7.707 EB", HumanSize(7.707*1000*1000*1000*1000*1000*1000)) +} + +func TestFromHumanSize(t *testing.T) { + assertFromHumanSize(t, "32", false, 32) + assertFromHumanSize(t, "32b", false, 32) + assertFromHumanSize(t, "32B", false, 32) + assertFromHumanSize(t, "32k", false, 32*1000) + assertFromHumanSize(t, "32K", false, 32*1000) + assertFromHumanSize(t, "32kb", false, 32*1000) + assertFromHumanSize(t, "32Kb", false, 32*1000) + assertFromHumanSize(t, "32Mb", false, 32*1000*1000) + assertFromHumanSize(t, "32Gb", false, 32*1000*1000*1000) + assertFromHumanSize(t, "32Tb", false, 32*1000*1000*1000*1000) + assertFromHumanSize(t, "8Pb", false, 8*1000*1000*1000*1000*1000) + + assertFromHumanSize(t, "", true, -1) + assertFromHumanSize(t, "hello", true, -1) + assertFromHumanSize(t, "-32", true, -1) + assertFromHumanSize(t, " 32 ", true, -1) + assertFromHumanSize(t, "32 mb", true, -1) + assertFromHumanSize(t, "32m b", true, -1) + assertFromHumanSize(t, "32bm", true, -1) +} + +func assertFromHumanSize(t *testing.T, size string, expectError bool, expectedBytes int64) { + actualBytes, err := FromHumanSize(size) + if (err != nil) && !expectError { + t.Errorf("Unexpected error parsing '%s': %s", size, err) + } + if (err == nil) && expectError { + t.Errorf("Expected to get an error parsing '%s', but got none (bytes=%d)", size, actualBytes) + } + if actualBytes != expectedBytes { + t.Errorf("Expected '%s' to parse as %d bytes, got %d", size, expectedBytes, actualBytes) + } +} + +func TestRAMInBytes(t *testing.T) { + assertRAMInBytes(t, "32", false, 32) + assertRAMInBytes(t, "32b", false, 32) + assertRAMInBytes(t, "32B", false, 32) + assertRAMInBytes(t, "32k", false, 32*1024) + assertRAMInBytes(t, "32K", false, 32*1024) + assertRAMInBytes(t, "32kb", false, 32*1024) + assertRAMInBytes(t, "32Kb", false, 32*1024) + assertRAMInBytes(t, "32Mb", false, 32*1024*1024) + assertRAMInBytes(t, "32MB", false, 32*1024*1024) + assertRAMInBytes(t, "32Gb", false, 32*1024*1024*1024) + assertRAMInBytes(t, "32G", false, 32*1024*1024*1024) + assertRAMInBytes(t, "32Tb", false, 32*1024*1024*1024*1024) + + assertRAMInBytes(t, "", true, -1) + assertRAMInBytes(t, "hello", true, -1) + assertRAMInBytes(t, "-32", true, -1) + assertRAMInBytes(t, " 32 ", true, -1) + assertRAMInBytes(t, "32 mb", true, -1) + assertRAMInBytes(t, "32m b", true, -1) + assertRAMInBytes(t, "32bm", true, -1) +} + +func assertRAMInBytes(t *testing.T, size string, expectError bool, expectedBytes int64) { + actualBytes, err := RAMInBytes(size) + if (err != nil) && !expectError { + t.Errorf("Unexpected error parsing '%s': %s", size, err) + } + if (err == nil) && expectError { + t.Errorf("Expected to get an error parsing '%s', but got none (bytes=%d)", size, actualBytes) + } + if actualBytes != expectedBytes { + t.Errorf("Expected '%s' to parse as %d bytes, got %d", size, expectedBytes, actualBytes) + } +} diff --git a/manager/container.go b/manager/container.go index 904bb8e6..856b6447 100644 --- a/manager/container.go +++ b/manager/container.go @@ -20,6 +20,7 @@ import ( "sync" "time" + "github.com/docker/docker/pkg/units" "github.com/golang/glog" "github.com/google/cadvisor/container" "github.com/google/cadvisor/info" @@ -49,6 +50,9 @@ type containerData struct { lock sync.Mutex housekeepingInterval time.Duration + // Whether to log the usage of this container when it is updated. + logUsage bool + // Tells the container to stop. stop chan bool } @@ -81,25 +85,27 @@ func (c *containerData) GetInfo() (*containerInfo, error) { return &c.info, nil } -func newContainerData(containerName string, driver storage.StorageDriver, handler container.ContainerHandler) (*containerData, error) { +func newContainerData(containerName string, driver storage.StorageDriver, handler container.ContainerHandler, logUsage bool) (*containerData, error) { if driver == nil { return nil, fmt.Errorf("nil storage driver") } if handler == nil { return nil, fmt.Errorf("nil container handler") } - - cont := &containerData{} - cont.handler = handler ref, err := handler.ContainerReference() if err != nil { return nil, err } + + cont := &containerData{ + handler: handler, + storageDriver: driver, + housekeepingInterval: *HousekeepingInterval, + logUsage: logUsage, + stop: make(chan bool, 1), + } cont.info.Name = ref.Name cont.info.Aliases = ref.Aliases - cont.storageDriver = driver - cont.housekeepingInterval = *HousekeepingInterval - cont.stop = make(chan bool, 1) return cont, nil } @@ -153,7 +159,31 @@ func (c *containerData) housekeeping() { // Log if housekeeping took too long. duration := time.Since(start) if duration >= longHousekeeping { - glog.V(2).Infof("Housekeeping(%s) took %s", c.info.Name, duration) + glog.V(2).Infof("[%s] Housekeeping took %s", c.info.Name, duration) + } + } + + // Log usage if asked to do so. + if c.logUsage { + stats, err := c.storageDriver.RecentStats(c.info.Name, 2) + if err != nil { + glog.Infof("[%s] Failed to get recent stats for logging usage: %v", c.info.Name, err) + } else if len(stats) < 2 { + // Ignore, not enough stats yet. + } else { + usageCpuNs := uint64(0) + usageMemory := uint64(0) + + if stats[0].Cpu != nil && stats[1].Cpu != nil { + usageCpuNs = stats[1].Cpu.Usage.Total - stats[0].Cpu.Usage.Total + } + if stats[1].Memory != nil { + usageMemory = stats[1].Memory.Usage + } + + usageInCores := float64(usageCpuNs) / float64(stats[1].Timestamp.Sub(stats[0].Timestamp).Nanoseconds()) + usageInHuman := units.HumanSize(int64(usageMemory)) + glog.Infof("[%s] %.3f cores, %s of memory", c.info.Name, usageInCores, usageInHuman) } } diff --git a/manager/container_test.go b/manager/container_test.go index 2166f297..32f2d1ae 100644 --- a/manager/container_test.go +++ b/manager/container_test.go @@ -34,7 +34,7 @@ const containerName = "/container" func newTestContainerData(t *testing.T) (*containerData, *container.MockContainerHandler, *stest.MockStorageDriver) { mockHandler := container.NewMockContainerHandler(containerName) mockDriver := &stest.MockStorageDriver{} - ret, err := newContainerData(containerName, mockDriver, mockHandler) + ret, err := newContainerData(containerName, mockDriver, mockHandler, false) if err != nil { t.Fatal(err) } diff --git a/manager/manager.go b/manager/manager.go index b6cccecf..6a07864d 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -23,6 +23,7 @@ import ( "sync" "time" + "github.com/docker/libcontainer/cgroups" "github.com/golang/glog" "github.com/google/cadvisor/container" "github.com/google/cadvisor/info" @@ -30,6 +31,7 @@ import ( ) var globalHousekeepingInterval = flag.Duration("global_housekeeping_interval", 1*time.Minute, "Interval between global housekeepings") +var logCadvisorUsage = flag.Bool("log_cadvisor_usage", false, "Whether to log the usage of the cAdvisor container") // The Manager interface defines operations for starting a manager and getting // container and machine information. @@ -58,10 +60,19 @@ func New(driver storage.StorageDriver) (Manager, error) { if driver == nil { return nil, fmt.Errorf("nil storage driver!") } + + // Detect the container we are running on. + selfContainer, err := cgroups.GetThisCgroupDir("cpu") + if err != nil { + return nil, err + } + glog.Infof("cAdvisor running in container: %q", selfContainer) + newManager := &manager{ - containers: make(map[string]*containerData), - quitChannels: make([]chan error, 0, 2), - storageDriver: driver, + containers: make(map[string]*containerData), + quitChannels: make([]chan error, 0, 2), + storageDriver: driver, + cadvisorContainer: selfContainer, } machineInfo, err := getMachineInfo() @@ -83,12 +94,13 @@ func New(driver storage.StorageDriver) (Manager, error) { } type manager struct { - containers map[string]*containerData - containersLock sync.RWMutex - storageDriver storage.StorageDriver - machineInfo info.MachineInfo - versionInfo info.VersionInfo - quitChannels []chan error + containers map[string]*containerData + containersLock sync.RWMutex + storageDriver storage.StorageDriver + machineInfo info.MachineInfo + versionInfo info.VersionInfo + quitChannels []chan error + cadvisorContainer string } // Start the container manager. @@ -270,7 +282,8 @@ func (m *manager) createContainer(containerName string) error { if err != nil { return err } - cont, err := newContainerData(containerName, m.storageDriver, handler) + logUsage := *logCadvisorUsage && containerName == m.cadvisorContainer + cont, err := newContainerData(containerName, m.storageDriver, handler, logUsage) if err != nil { return err } @@ -280,7 +293,7 @@ func (m *manager) createContainer(containerName string) error { m.containersLock.Lock() defer m.containersLock.Unlock() - // Check that the container didn't already exist\ + // Check that the container didn't already exist. _, ok := m.containers[containerName] if ok { return true diff --git a/manager/manager_test.go b/manager/manager_test.go index 1754a7b4..4e682f59 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -44,7 +44,7 @@ func createManagerAndAddContainers( if ret, ok := mif.(*manager); ok { for _, name := range containers { mockHandler := container.NewMockContainerHandler(name) - ret.containers[name], err = newContainerData(name, driver, mockHandler) + ret.containers[name], err = newContainerData(name, driver, mockHandler, false) if err != nil { t.Fatal(err) }