Add metrics caching

collector/collector_manager.go

@@ -22,12 +22,11 @@ import (
 	"github.com/google/cadvisor/info/v1"
 )
 
-type collectorManager struct {
+type GenericCollectorManager struct {
-	collectors []*collectorData
+	Collectors         []*collectorData
+	NextCollectionTime time.Time
 }
 
-var _ CollectorManager = &collectorManager{}
-
 type collectorData struct {
 	collector          Collector
 	nextCollectionTime time.Time
@@ -35,26 +34,27 @@ type collectorData struct {
 
 // Returns a new CollectorManager that is thread-compatible.
 func NewCollectorManager() (CollectorManager, error) {
-	return &collectorManager{
+	return &GenericCollectorManager{
-		collectors: []*collectorData{},
+		Collectors:         []*collectorData{},
+		NextCollectionTime: time.Now(),
 	}, nil
 }
 
-func (cm *collectorManager) RegisterCollector(collector Collector) error {
+func (cm *GenericCollectorManager) RegisterCollector(collector Collector) error {
-	cm.collectors = append(cm.collectors, &collectorData{
+	cm.Collectors = append(cm.Collectors, &collectorData{
 		collector:          collector,
 		nextCollectionTime: time.Now(),
 	})
 	return nil
 }
 
-func (cm *collectorManager) Collect() (time.Time, []v1.Metric, error) {
+func (cm *GenericCollectorManager) Collect() (time.Time, []v1.Metric, error) {
 	var errors []error
 
 	// Collect from all collectors that are ready.
 	var next time.Time
 	var metrics []v1.Metric
-	for _, c := range cm.collectors {
+	for _, c := range cm.Collectors {
 		if c.nextCollectionTime.Before(time.Now()) {
 			nextCollection, newMetrics, err := c.collector.Collect()
 			if err != nil {
@@ -69,7 +69,7 @@ func (cm *collectorManager) Collect() (time.Time, []v1.Metric, error) {
 			next = c.nextCollectionTime
 		}
 	}
-
+	cm.NextCollectionTime = next
 	return next, metrics, compileErrors(errors)
 }
 

collector/collector_manager_test.go

@@ -38,7 +38,7 @@ func (fc *fakeCollector) Name() string {
 }
 
 func TestCollect(t *testing.T) {
-	cm := &collectorManager{}
+	cm := &GenericCollectorManager{}
 
 	firstTime := time.Now().Add(-time.Hour)
 	secondTime := time.Now().Add(time.Hour)

manager/container.go

@@ -340,19 +340,7 @@ func (c *containerData) housekeeping() {
 			}
 		}
 
-		// TODO(vmarmol): Export metrics.
+		next := c.nextHousekeeping(lastHousekeeping)
-		// Run custom collectors.
-		nextCollectionTime, _, err := c.collectorManager.Collect()
-		if err != nil && c.allowErrorLogging() {
-			glog.Warningf("[%s] Collection failed: %v", c.info.Name, err)
-		}
-
-		// Next housekeeping is the first of the stats or the custom collector's housekeeping.
-		nextHousekeeping := c.nextHousekeeping(lastHousekeeping)
-		next := nextHousekeeping
-		if !nextCollectionTime.IsZero() && nextCollectionTime.Before(nextHousekeeping) {
-			next = nextCollectionTime
-		}
 
 		// Schedule the next housekeeping. Sleep until that time.
 		if time.Now().Before(next) {
@@ -432,6 +420,20 @@ func (c *containerData) updateStats() error {
 			glog.V(2).Infof("Failed to add summary stats for %q: %v", c.info.Name, err)
 		}
 	}
+	var customStatsErr error
+	if c.collectorManager != nil {
+		cm := c.collectorManager.(*collector.GenericCollectorManager)
+		if cm.NextCollectionTime.Before(time.Now()) {
+			customStats, err := c.updateCustomStats()
+			if customStats != nil {
+				stats.CustomMetrics = customStats
+			}
+			if err != nil {
+				customStatsErr = err
+			}
+		}
+	}
+
 	ref, err := c.handler.ContainerReference()
 	if err != nil {
 		// Ignore errors if the container is dead.
@@ -444,7 +446,21 @@ func (c *containerData) updateStats() error {
 	if err != nil {
 		return err
 	}
+	if statsErr != nil {
 		return statsErr
+	}
+	return customStatsErr
+}
+
+func (c *containerData) updateCustomStats() ([]info.Metric, error) {
+	_, customStats, customStatsErr := c.collectorManager.Collect()
+	if customStatsErr != nil {
+		if !c.handler.Exists() {
+			return customStats, nil
+		}
+		customStatsErr = fmt.Errorf("%v, continuing to push custom stats", customStatsErr)
+	}
+	return customStats, customStatsErr
 }
 
 func (c *containerData) updateSubcontainers() error {

manager/container_test.go

@@ -41,7 +41,7 @@ func setupContainerData(t *testing.T, spec info.ContainerSpec) (*containerData,
 		nil,
 	)
 	memoryCache := memory.New(60, nil)
-	ret, err := newContainerData(containerName, memoryCache, mockHandler, nil, false, &collector.FakeCollectorManager{})
+	ret, err := newContainerData(containerName, memoryCache, mockHandler, nil, false, &collector.GenericCollectorManager{})
 	if err != nil {
 		t.Fatal(err)
 	}

manager/manager_test.go

@@ -53,7 +53,7 @@ func createManagerAndAddContainers(
 			spec,
 			nil,
 		).Once()
-		cont, err := newContainerData(name, memoryCache, mockHandler, nil, false, &collector.FakeCollectorManager{})
+		cont, err := newContainerData(name, memoryCache, mockHandler, nil, false, &collector.GenericCollectorManager{})
 		if err != nil {
 			t.Fatal(err)
 		}