prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Log one error message per hour for failing containers.

Browse Source
This commit is contained in:
Rohit Jnagal 2014-12-23 00:49:48 +00:00
parent 5994c17b1e
commit b0462e3e94
1 changed files with 20 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
manager/container.go
View File

@ -45,7 +45,7 @@ type containerData struct {
lock sync.Mutex lock sync.Mutex
housekeepingInterval time.Duration housekeepingInterval time.Duration
lastUpdatedTime time.Time lastUpdatedTime time.Time
lastErrorTime time.Time
// Whether to log the usage of this container when it is updated. // Whether to log the usage of this container when it is updated.
logUsage bool logUsage bool
@ -64,6 +64,14 @@ func (c *containerData) Stop() error {
return nil return nil
} }
func (c *containerData) allowErrorLogging() bool {
if !c.lastErrorTime.IsZero() && time.Since(c.lastErrorTime) > time.Hour {
c.lastErrorTime = time.Now()
return true
}
return false
}
func (c *containerData) GetInfo() (*containerInfo, error) { func (c *containerData) GetInfo() (*containerInfo, error) {
// Get spec and subcontainers. // Get spec and subcontainers.
if time.Since(c.lastUpdatedTime) > 5*time.Second { if time.Since(c.lastUpdatedTime) > 5*time.Second {
@ -112,7 +120,9 @@ func (self *containerData) nextHousekeeping(lastHousekeeping time.Time) time.Tim
if *allowDynamicHousekeeping { if *allowDynamicHousekeeping {
stats, err := self.storageDriver.RecentStats(self.info.Name, 2) stats, err := self.storageDriver.RecentStats(self.info.Name, 2)
if err != nil { if err != nil {
if self.allowErrorLogging() {
glog.Warningf("Failed to get RecentStats(%q) while determining the next housekeeping: %v", self.info.Name, err) glog.Warningf("Failed to get RecentStats(%q) while determining the next housekeeping: %v", self.info.Name, err)
}
} else if len(stats) == 2 { } else if len(stats) == 2 {
// TODO(vishnuk): Use no processes as a signal. // TODO(vishnuk): Use no processes as a signal.
// Raise the interval if usage hasn't changed in the last housekeeping. // Raise the interval if usage hasn't changed in the last housekeeping.
@ -164,7 +174,9 @@ func (c *containerData) housekeeping() {
if c.logUsage { if c.logUsage {
stats, err := c.storageDriver.RecentStats(c.info.Name, 2) stats, err := c.storageDriver.RecentStats(c.info.Name, 2)
if err != nil { if err != nil {
if c.allowErrorLogging() {
glog.Infof("[%s] Failed to get recent stats for logging usage: %v", c.info.Name, err) glog.Infof("[%s] Failed to get recent stats for logging usage: %v", c.info.Name, err)
}
} else if len(stats) < 2 { } else if len(stats) < 2 {
// Ignore, not enough stats yet. // Ignore, not enough stats yet.
} else { } else {
@ -189,9 +201,11 @@ func (c *containerData) housekeeping() {
func (c *containerData) housekeepingTick() { func (c *containerData) housekeepingTick() {
err := c.updateStats() err := c.updateStats()
if err != nil { if err != nil {
if c.allowErrorLogging() {
glog.Infof("Failed to update stats for container \"%s\": %s", c.info.Name, err) glog.Infof("Failed to update stats for container \"%s\": %s", c.info.Name, err)
} }
} }
}
func (c *containerData) updateSpec() error { func (c *containerData) updateSpec() error {
spec, err := c.handler.GetSpec() spec, err := c.handler.GetSpec()