prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding --log_cadvisor_usage flag to log the resource usage of cAdvisor.

Browse Source 
This helps during performance analysis.
This commit is contained in:
Victor Marmol 2014-10-14 03:54:17 -07:00
parent 22a55636f4
commit 62b37eca1d
10 changed files with 342 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Godeps/Godeps.json generated
View File

@ -5,6 +5,11 @@
"./..."
],
"Deps": [
{
"ImportPath": "code.google.com/p/go.exp/inotify",
"Comment": "null-75",
"Rev": "bd8df7009305d6ada223ea3c95b94c0f38bfa119"
},
{
"ImportPath": "code.google.com/p/goauth2/oauth",
"Comment": "weekly-56",
@ -43,6 +48,11 @@
"Comment": "v1.1.1-402-g487a417",
"Rev": "487a417d9fd074d0e78876072c7d1ebfd398ea7a"
},
{
"ImportPath": "github.com/docker/docker/pkg/units",
"Comment": "v1.1.1-402-g487a417",
"Rev": "487a417d9fd074d0e78876072c7d1ebfd398ea7a"
},
{
"ImportPath": "github.com/docker/libcontainer",
"Comment": "v1.2.0-46-g0da391f",

2
Godeps/_workspace/src/github.com/docker/docker/pkg/units/MAINTAINERS generated vendored Normal file
View File

@ -0,0 +1,2 @@
Michael Crosby <michael@crosbymichael.com> (@crosbymichael)
Victor Vieux <vieux@docker.com> (@vieux)

31
Godeps/_workspace/src/github.com/docker/docker/pkg/units/duration.go generated vendored Normal file
View File

@ -0,0 +1,31 @@
package units
import (
"fmt"
"time"
)
// HumanDuration returns a human-readable approximation of a duration
// (eg. "About a minute", "4 hours ago", etc.)
func HumanDuration(d time.Duration) string {
if seconds := int(d.Seconds()); seconds < 1 {
return "Less than a second"
} else if seconds < 60 {
return fmt.Sprintf("%d seconds", seconds)
} else if minutes := int(d.Minutes()); minutes == 1 {
return "About a minute"
} else if minutes < 60 {
return fmt.Sprintf("%d minutes", minutes)
} else if hours := int(d.Hours()); hours == 1 {
return "About an hour"
} else if hours < 48 {
return fmt.Sprintf("%d hours", hours)
} else if hours < 24*7*2 {
return fmt.Sprintf("%d days", hours/24)
} else if hours < 24*30*3 {
return fmt.Sprintf("%d weeks", hours/24/7)
} else if hours < 24*365*2 {
return fmt.Sprintf("%d months", hours/24/30)
}
return fmt.Sprintf("%f years", d.Hours()/24/365)
}

52
Godeps/_workspace/src/github.com/docker/docker/pkg/units/duration_test.go generated vendored Normal file
View File

@ -0,0 +1,52 @@
package units
import (
"testing"
"time"
)
func TestHumanDuration(t *testing.T) {
// Useful duration abstractions
day := 24 * time.Hour
week := 7 * day
month := 30 * day
year := 365 * day
assertEquals(t, "Less than a second", HumanDuration(450*time.Millisecond))
assertEquals(t, "47 seconds", HumanDuration(47*time.Second))
assertEquals(t, "About a minute", HumanDuration(1*time.Minute))
assertEquals(t, "3 minutes", HumanDuration(3*time.Minute))
assertEquals(t, "35 minutes", HumanDuration(35*time.Minute))
assertEquals(t, "35 minutes", HumanDuration(35*time.Minute+40*time.Second))
assertEquals(t, "About an hour", HumanDuration(1*time.Hour))
assertEquals(t, "About an hour", HumanDuration(1*time.Hour+45*time.Minute))
assertEquals(t, "3 hours", HumanDuration(3*time.Hour))
assertEquals(t, "3 hours", HumanDuration(3*time.Hour+59*time.Minute))
assertEquals(t, "4 hours", HumanDuration(3*time.Hour+60*time.Minute))
assertEquals(t, "24 hours", HumanDuration(24*time.Hour))
assertEquals(t, "36 hours", HumanDuration(1*day+12*time.Hour))
assertEquals(t, "2 days", HumanDuration(2*day))
assertEquals(t, "7 days", HumanDuration(7*day))
assertEquals(t, "13 days", HumanDuration(13*day+5*time.Hour))
assertEquals(t, "2 weeks", HumanDuration(2*week))
assertEquals(t, "2 weeks", HumanDuration(2*week+4*day))
assertEquals(t, "3 weeks", HumanDuration(3*week))
assertEquals(t, "4 weeks", HumanDuration(4*week))
assertEquals(t, "4 weeks", HumanDuration(4*week+3*day))
assertEquals(t, "4 weeks", HumanDuration(1*month))
assertEquals(t, "6 weeks", HumanDuration(1*month+2*week))
assertEquals(t, "8 weeks", HumanDuration(2*month))
assertEquals(t, "3 months", HumanDuration(3*month+1*week))
assertEquals(t, "5 months", HumanDuration(5*month+2*week))
assertEquals(t, "13 months", HumanDuration(13*month))
assertEquals(t, "23 months", HumanDuration(23*month))
assertEquals(t, "24 months", HumanDuration(24*month))
assertEquals(t, "2.010959 years", HumanDuration(24*month+2*week))
assertEquals(t, "3.164384 years", HumanDuration(3*year+2*month))
}
func assertEquals(t *testing.T, expected, actual interface{}) {
if expected != actual {
t.Errorf("Expected '%s' but got '%s'", expected, actual)
}
}

94
Godeps/_workspace/src/github.com/docker/docker/pkg/units/size.go generated vendored Normal file
View File

@ -0,0 +1,94 @@
package units
import (
"fmt"
"regexp"
"strconv"
"strings"
)
// HumanSize returns a human-readable approximation of a size
// using SI standard (eg. "44kB", "17MB")
func HumanSize(size int64) string {
i := 0
var sizef float64
sizef = float64(size)
units := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
for sizef >= 1000.0 {
sizef = sizef / 1000.0
i++
}
return fmt.Sprintf("%.4g %s", sizef, units[i])
}
// FromHumanSize returns an integer from a human-readable specification of a size
// using SI standard (eg. "44kB", "17MB")
func FromHumanSize(size string) (int64, error) {
re, err := regexp.Compile("^(\\d+)([kKmMgGtTpP])?[bB]?$")
if err != nil {
return -1, fmt.Errorf("%s does not specify not a size", size)
}
matches := re.FindStringSubmatch(size)
if len(matches) != 3 {
return -1, fmt.Errorf("Invalid size: '%s'", size)
}
theSize, err := strconv.ParseInt(matches[1], 10, 0)
if err != nil {
return -1, err
}
unit := strings.ToLower(matches[2])
if unit == "k" {
theSize *= 1000
} else if unit == "m" {
theSize *= 1000 * 1000
} else if unit == "g" {
theSize *= 1000 * 1000 * 1000
} else if unit == "t" {
theSize *= 1000 * 1000 * 1000 * 1000
} else if unit == "p" {
theSize *= 1000 * 1000 * 1000 * 1000 * 1000
}
return theSize, nil
}
// Parses a human-readable string representing an amount of RAM
// in bytes, kibibytes, mebibytes, gibibytes, or tebibytes and
// returns the number of bytes, or -1 if the string is unparseable.
// Units are case-insensitive, and the 'b' suffix is optional.
func RAMInBytes(size string) (int64, error) {
re, err := regexp.Compile("^(\\d+)([kKmMgGtT])?[bB]?$")
if err != nil {
return -1, err
}
matches := re.FindStringSubmatch(size)
if len(matches) != 3 {
return -1, fmt.Errorf("Invalid size: '%s'", size)
}
memLimit, err := strconv.ParseInt(matches[1], 10, 0)
if err != nil {
return -1, err
}
unit := strings.ToLower(matches[2])
if unit == "k" {
memLimit *= 1024
} else if unit == "m" {
memLimit *= 1024 * 1024
} else if unit == "g" {
memLimit *= 1024 * 1024 * 1024
} else if unit == "t" {
memLimit *= 1024 * 1024 * 1024 * 1024
}
return memLimit, nil
}

89
Godeps/_workspace/src/github.com/docker/docker/pkg/units/size_test.go generated vendored Normal file
View File

@ -0,0 +1,89 @@
package units
import (
"testing"
)
func TestHumanSize(t *testing.T) {
assertEquals(t, "1 kB", HumanSize(1000))
assertEquals(t, "1.024 kB", HumanSize(1024))
assertEquals(t, "1 MB", HumanSize(1000000))
assertEquals(t, "1.049 MB", HumanSize(1048576))
assertEquals(t, "2 MB", HumanSize(2*1000*1000))
assertEquals(t, "3.42 GB", HumanSize(3.42*1000*1000*1000))
assertEquals(t, "5.372 TB", HumanSize(5.372*1000*1000*1000*1000))
assertEquals(t, "2.22 PB", HumanSize(2.22*1000*1000*1000*1000*1000))
assertEquals(t, "2.22 EB", HumanSize(2.22*1000*1000*1000*1000*1000*1000))
assertEquals(t, "7.707 EB", HumanSize(7.707*1000*1000*1000*1000*1000*1000))
}
func TestFromHumanSize(t *testing.T) {
assertFromHumanSize(t, "32", false, 32)
assertFromHumanSize(t, "32b", false, 32)
assertFromHumanSize(t, "32B", false, 32)
assertFromHumanSize(t, "32k", false, 32*1000)
assertFromHumanSize(t, "32K", false, 32*1000)
assertFromHumanSize(t, "32kb", false, 32*1000)
assertFromHumanSize(t, "32Kb", false, 32*1000)
assertFromHumanSize(t, "32Mb", false, 32*1000*1000)
assertFromHumanSize(t, "32Gb", false, 32*1000*1000*1000)
assertFromHumanSize(t, "32Tb", false, 32*1000*1000*1000*1000)
assertFromHumanSize(t, "8Pb", false, 8*1000*1000*1000*1000*1000)
assertFromHumanSize(t, "", true, -1)
assertFromHumanSize(t, "hello", true, -1)
assertFromHumanSize(t, "-32", true, -1)
assertFromHumanSize(t, " 32 ", true, -1)
assertFromHumanSize(t, "32 mb", true, -1)
assertFromHumanSize(t, "32m b", true, -1)
assertFromHumanSize(t, "32bm", true, -1)
}
func assertFromHumanSize(t *testing.T, size string, expectError bool, expectedBytes int64) {
actualBytes, err := FromHumanSize(size)
if (err != nil) && !expectError {
t.Errorf("Unexpected error parsing '%s': %s", size, err)
}
if (err == nil) && expectError {
t.Errorf("Expected to get an error parsing '%s', but got none (bytes=%d)", size, actualBytes)
}
if actualBytes != expectedBytes {
t.Errorf("Expected '%s' to parse as %d bytes, got %d", size, expectedBytes, actualBytes)
}
}
func TestRAMInBytes(t *testing.T) {
assertRAMInBytes(t, "32", false, 32)
assertRAMInBytes(t, "32b", false, 32)
assertRAMInBytes(t, "32B", false, 32)
assertRAMInBytes(t, "32k", false, 32*1024)
assertRAMInBytes(t, "32K", false, 32*1024)
assertRAMInBytes(t, "32kb", false, 32*1024)
assertRAMInBytes(t, "32Kb", false, 32*1024)
assertRAMInBytes(t, "32Mb", false, 32*1024*1024)
assertRAMInBytes(t, "32MB", false, 32*1024*1024)
assertRAMInBytes(t, "32Gb", false, 32*1024*1024*1024)
assertRAMInBytes(t, "32G", false, 32*1024*1024*1024)
assertRAMInBytes(t, "32Tb", false, 32*1024*1024*1024*1024)
assertRAMInBytes(t, "", true, -1)
assertRAMInBytes(t, "hello", true, -1)
assertRAMInBytes(t, "-32", true, -1)
assertRAMInBytes(t, " 32 ", true, -1)
assertRAMInBytes(t, "32 mb", true, -1)
assertRAMInBytes(t, "32m b", true, -1)
assertRAMInBytes(t, "32bm", true, -1)
}
func assertRAMInBytes(t *testing.T, size string, expectError bool, expectedBytes int64) {
actualBytes, err := RAMInBytes(size)
if (err != nil) && !expectError {
t.Errorf("Unexpected error parsing '%s': %s", size, err)
}
if (err == nil) && expectError {
t.Errorf("Expected to get an error parsing '%s', but got none (bytes=%d)", size, actualBytes)
}
if actualBytes != expectedBytes {
t.Errorf("Expected '%s' to parse as %d bytes, got %d", size, expectedBytes, actualBytes)
}
}

46
manager/container.go
View File

@ -20,6 +20,7 @@ import (
"sync"
"time"
"github.com/docker/docker/pkg/units"
"github.com/golang/glog"
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/info"
@ -49,6 +50,9 @@ type containerData struct {
lock sync.Mutex
housekeepingInterval time.Duration
// Whether to log the usage of this container when it is updated.
logUsage bool
// Tells the container to stop.
stop chan bool
}
@ -81,25 +85,27 @@ func (c *containerData) GetInfo() (*containerInfo, error) {
return &c.info, nil
}
func newContainerData(containerName string, driver storage.StorageDriver, handler container.ContainerHandler) (*containerData, error) {
func newContainerData(containerName string, driver storage.StorageDriver, handler container.ContainerHandler, logUsage bool) (*containerData, error) {
if driver == nil {
return nil, fmt.Errorf("nil storage driver")
}
if handler == nil {
return nil, fmt.Errorf("nil container handler")
}
cont := &containerData{}
cont.handler = handler
ref, err := handler.ContainerReference()
if err != nil {
return nil, err
}
cont := &containerData{
handler: handler,
storageDriver: driver,
housekeepingInterval: *HousekeepingInterval,
logUsage: logUsage,
stop: make(chan bool, 1),
}
cont.info.Name = ref.Name
cont.info.Aliases = ref.Aliases
cont.storageDriver = driver
cont.housekeepingInterval = *HousekeepingInterval
cont.stop = make(chan bool, 1)
return cont, nil
}
@ -153,7 +159,31 @@ func (c *containerData) housekeeping() {
// Log if housekeeping took too long.
duration := time.Since(start)
if duration >= longHousekeeping {
glog.V(2).Infof("Housekeeping(%s) took %s", c.info.Name, duration)
glog.V(2).Infof("[%s] Housekeeping took %s", c.info.Name, duration)
}
}
// Log usage if asked to do so.
if c.logUsage {
stats, err := c.storageDriver.RecentStats(c.info.Name, 2)
if err != nil {
glog.Infof("[%s] Failed to get recent stats for logging usage: %v", c.info.Name, err)
} else if len(stats) < 2 {
// Ignore, not enough stats yet.
} else {
usageCpuNs := uint64(0)
usageMemory := uint64(0)
if stats[0].Cpu != nil && stats[1].Cpu != nil {
usageCpuNs = stats[1].Cpu.Usage.Total - stats[0].Cpu.Usage.Total
}
if stats[1].Memory != nil {
usageMemory = stats[1].Memory.Usage
}
usageInCores := float64(usageCpuNs) / float64(stats[1].Timestamp.Sub(stats[0].Timestamp).Nanoseconds())
usageInHuman := units.HumanSize(int64(usageMemory))
glog.Infof("[%s] %.3f cores, %s of memory", c.info.Name, usageInCores, usageInHuman)
}
}

2
manager/container_test.go
View File

@ -34,7 +34,7 @@ const containerName = "/container"
func newTestContainerData(t *testing.T) (*containerData, *container.MockContainerHandler, *stest.MockStorageDriver) {
mockHandler := container.NewMockContainerHandler(containerName)
mockDriver := &stest.MockStorageDriver{}
ret, err := newContainerData(containerName, mockDriver, mockHandler)
ret, err := newContainerData(containerName, mockDriver, mockHandler, false)
if err != nil {
t.Fatal(err)
}

35
manager/manager.go
View File

@ -23,6 +23,7 @@ import (
"sync"
"time"
"github.com/docker/libcontainer/cgroups"
"github.com/golang/glog"
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/info"
@ -30,6 +31,7 @@ import (
)
var globalHousekeepingInterval = flag.Duration("global_housekeeping_interval", 1*time.Minute, "Interval between global housekeepings")
var logCadvisorUsage = flag.Bool("log_cadvisor_usage", false, "Whether to log the usage of the cAdvisor container")
// The Manager interface defines operations for starting a manager and getting
// container and machine information.
@ -58,10 +60,19 @@ func New(driver storage.StorageDriver) (Manager, error) {
if driver == nil {
return nil, fmt.Errorf("nil storage driver!")
}
// Detect the container we are running on.
selfContainer, err := cgroups.GetThisCgroupDir("cpu")
if err != nil {
return nil, err
}
glog.Infof("cAdvisor running in container: %q", selfContainer)
newManager := &manager{
containers: make(map[string]*containerData),
quitChannels: make([]chan error, 0, 2),
storageDriver: driver,
containers: make(map[string]*containerData),
quitChannels: make([]chan error, 0, 2),
storageDriver: driver,
cadvisorContainer: selfContainer,
}
machineInfo, err := getMachineInfo()
@ -83,12 +94,13 @@ func New(driver storage.StorageDriver) (Manager, error) {
}
type manager struct {
containers map[string]*containerData
containersLock sync.RWMutex
storageDriver storage.StorageDriver
machineInfo info.MachineInfo
versionInfo info.VersionInfo
quitChannels []chan error
containers map[string]*containerData
containersLock sync.RWMutex
storageDriver storage.StorageDriver
machineInfo info.MachineInfo
versionInfo info.VersionInfo
quitChannels []chan error
cadvisorContainer string
}
// Start the container manager.
@ -270,7 +282,8 @@ func (m *manager) createContainer(containerName string) error {
if err != nil {
return err
}
cont, err := newContainerData(containerName, m.storageDriver, handler)
logUsage := *logCadvisorUsage && containerName == m.cadvisorContainer
cont, err := newContainerData(containerName, m.storageDriver, handler, logUsage)
if err != nil {
return err
}
@ -280,7 +293,7 @@ func (m *manager) createContainer(containerName string) error {
m.containersLock.Lock()
defer m.containersLock.Unlock()
// Check that the container didn't already exist\
// Check that the container didn't already exist.
_, ok := m.containers[containerName]
if ok {
return true

2
manager/manager_test.go
View File

@ -44,7 +44,7 @@ func createManagerAndAddContainers(
if ret, ok := mif.(*manager); ok {
for _, name := range containers {
mockHandler := container.NewMockContainerHandler(name)
ret.containers[name], err = newContainerData(name, driver, mockHandler)
ret.containers[name], err = newContainerData(name, driver, mockHandler, false)
if err != nil {
t.Fatal(err)
}