prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make housekeeping interval dynamic, fist signal is resource usage.

Browse Source 
If no resources are used by the container since the last housekeeping,
we double the housekeeping interval until --max_housekeeping_interval.
If usage was detected, we drop it back to the baseline
(--housekeeping_interval).

From my tests this reduces CPU usage in CoreOS from ~8% to ~3% with no
real loss of accuracy.

Fixes #159
This commit is contained in:
Victor Marmol 2014-09-10 10:48:47 -07:00
parent 4acd49ec83
commit 4e028d7995
2 changed files with 33 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
info/container.go
View File

@ -330,6 +330,11 @@ func (a *ContainerStats) Eq(b *ContainerStats) bool {
if !timeEq(a.Timestamp, b.Timestamp, timePrecision) {
return false
}
return a.StatsEq(b)
}
// Checks equality of the stats values.
func (a *ContainerStats) StatsEq(b *ContainerStats) bool {
if !reflect.DeepEqual(a.Cpu, b.Cpu) {
return false
}

37
manager/container.go
View File

@ -30,6 +30,7 @@ import (
// Housekeeping interval.
var HousekeepingInterval = flag.Duration("housekeeping_interval", 1*time.Second, "Interval between container housekeepings")
var maxHousekeepingInterval = flag.Duration("max_housekeeping_interval", 60*time.Second, "Largest interval to allow between container housekeepings")
// Internal mirror of the external data structure.
type containerStat struct {
@ -43,10 +44,11 @@ type containerInfo struct {
}
type containerData struct {
handler container.ContainerHandler
info containerInfo
storageDriver storage.StorageDriver
lock sync.Mutex
handler container.ContainerHandler
info containerInfo
storageDriver storage.StorageDriver
lock sync.Mutex
housekeepingInterval time.Duration
// Tells the container to stop.
stop chan bool
@ -98,6 +100,7 @@ func NewContainerData(containerName string, driver storage.StorageDriver) (*cont
cont.info.Name = ref.Name
cont.info.Aliases = ref.Aliases
cont.storageDriver = driver
cont.housekeepingInterval = *HousekeepingInterval
cont.stop = make(chan bool, 1)
return cont, nil
@ -105,10 +108,26 @@ func NewContainerData(containerName string, driver storage.StorageDriver) (*cont
// Determine when the next housekeeping should occur.
func (self *containerData) nextHousekeeping(lastHousekeeping time.Time) time.Time {
// For now, we just want to housekeep even HousekeepingInterval.
// TODO(vmarmol): Housekeep less if there are no change in stats.
// TODO(vishnuk): Housekeep less if there are no processes.
return lastHousekeeping.Add(*HousekeepingInterval)
stats, err := self.storageDriver.RecentStats(self.info.Name, 2)
if err != nil {
glog.Warningf("Failed to get RecentStats(%q) while determining the next housekeeping: %v", self.info.Name, err)
} else if len(stats) == 2 {
// TODO(vishnuk): Use no processes as a signal.
// Raise the interval if usage hasn't changed in the last housekeeping.
if stats[0].StatsEq(stats[1]) && (self.housekeepingInterval < *maxHousekeepingInterval) {
self.housekeepingInterval *= 2
if self.housekeepingInterval > *maxHousekeepingInterval {
self.housekeepingInterval = *maxHousekeepingInterval
}
glog.V(2).Infof("Raising housekeeping interval for %q to %v", self.info.Name, self.housekeepingInterval)
} else if self.housekeepingInterval != *HousekeepingInterval {
// Lower interval back to the baseline.
self.housekeepingInterval = *HousekeepingInterval
glog.V(1).Infof("Lowering housekeeping interval for %q to %v", self.info.Name, self.housekeepingInterval)
}
}
return lastHousekeeping.Add(self.housekeepingInterval)
}
func (c *containerData) housekeeping() {
@ -134,7 +153,7 @@ func (c *containerData) housekeeping() {
// Log if housekeeping took too long.
duration := time.Since(start)
if duration >= longHousekeeping {
glog.V(1).Infof("Housekeeping(%s) took %s", c.info.Name, duration)
glog.V(2).Infof("Housekeeping(%s) took %s", c.info.Name, duration)
}
}