prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make housekeeping interval dynamic, fist signal is resource usage.

Browse Source 
If no resources are used by the container since the last housekeeping,
we double the housekeeping interval until --max_housekeeping_interval.
If usage was detected, we drop it back to the baseline
(--housekeeping_interval).

From my tests this reduces CPU usage in CoreOS from ~8% to ~3% with no
real loss of accuracy.

Fixes #159
This commit is contained in:
Victor Marmol 2014-09-10 10:48:47 -07:00
parent 4acd49ec83
commit 4e028d7995
2 changed files with 33 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
info/container.go
View File

@ -330,6 +330,11 @@ func (a *ContainerStats) Eq(b *ContainerStats) bool {
if !timeEq(a.Timestamp, b.Timestamp, timePrecision) { if !timeEq(a.Timestamp, b.Timestamp, timePrecision) {
return false return false
} }
return a.StatsEq(b)
}
// Checks equality of the stats values.
func (a *ContainerStats) StatsEq(b *ContainerStats) bool {
if !reflect.DeepEqual(a.Cpu, b.Cpu) { if !reflect.DeepEqual(a.Cpu, b.Cpu) {
return false return false
} }

37
manager/container.go
View File

@ -30,6 +30,7 @@ import (
// Housekeeping interval. // Housekeeping interval.
var HousekeepingInterval = flag.Duration("housekeeping_interval", 1*time.Second, "Interval between container housekeepings") var HousekeepingInterval = flag.Duration("housekeeping_interval", 1*time.Second, "Interval between container housekeepings")
var maxHousekeepingInterval = flag.Duration("max_housekeeping_interval", 60*time.Second, "Largest interval to allow between container housekeepings")
// Internal mirror of the external data structure. // Internal mirror of the external data structure.
type containerStat struct { type containerStat struct {
@ -43,10 +44,11 @@ type containerInfo struct {
} }
type containerData struct { type containerData struct {
handler container.ContainerHandler handler container.ContainerHandler
info containerInfo info containerInfo
storageDriver storage.StorageDriver storageDriver storage.StorageDriver
lock sync.Mutex lock sync.Mutex
housekeepingInterval time.Duration
// Tells the container to stop. // Tells the container to stop.
stop chan bool stop chan bool
@ -98,6 +100,7 @@ func NewContainerData(containerName string, driver storage.StorageDriver) (*cont
cont.info.Name = ref.Name cont.info.Name = ref.Name
cont.info.Aliases = ref.Aliases cont.info.Aliases = ref.Aliases
cont.storageDriver = driver cont.storageDriver = driver
cont.housekeepingInterval = *HousekeepingInterval
cont.stop = make(chan bool, 1) cont.stop = make(chan bool, 1)
return cont, nil return cont, nil
@ -105,10 +108,26 @@ func NewContainerData(containerName string, driver storage.StorageDriver) (*cont
// Determine when the next housekeeping should occur. // Determine when the next housekeeping should occur.
func (self *containerData) nextHousekeeping(lastHousekeeping time.Time) time.Time { func (self *containerData) nextHousekeeping(lastHousekeeping time.Time) time.Time {
// For now, we just want to housekeep even HousekeepingInterval. stats, err := self.storageDriver.RecentStats(self.info.Name, 2)
// TODO(vmarmol): Housekeep less if there are no change in stats. if err != nil {
// TODO(vishnuk): Housekeep less if there are no processes. glog.Warningf("Failed to get RecentStats(%q) while determining the next housekeeping: %v", self.info.Name, err)
return lastHousekeeping.Add(*HousekeepingInterval) } else if len(stats) == 2 {
// TODO(vishnuk): Use no processes as a signal.
// Raise the interval if usage hasn't changed in the last housekeeping.
if stats[0].StatsEq(stats[1]) && (self.housekeepingInterval < *maxHousekeepingInterval) {
self.housekeepingInterval *= 2
if self.housekeepingInterval > *maxHousekeepingInterval {
self.housekeepingInterval = *maxHousekeepingInterval
}
glog.V(2).Infof("Raising housekeeping interval for %q to %v", self.info.Name, self.housekeepingInterval)
} else if self.housekeepingInterval != *HousekeepingInterval {
// Lower interval back to the baseline.
self.housekeepingInterval = *HousekeepingInterval
glog.V(1).Infof("Lowering housekeeping interval for %q to %v", self.info.Name, self.housekeepingInterval)
}
}
return lastHousekeeping.Add(self.housekeepingInterval)
} }
func (c *containerData) housekeeping() { func (c *containerData) housekeeping() {
@ -134,7 +153,7 @@ func (c *containerData) housekeeping() {
// Log if housekeeping took too long. // Log if housekeeping took too long.
duration := time.Since(start) duration := time.Since(start)
if duration >= longHousekeeping { if duration >= longHousekeeping {
glog.V(1).Infof("Housekeeping(%s) took %s", c.info.Name, duration) glog.V(2).Infof("Housekeeping(%s) took %s", c.info.Name, duration)
} }
} }