Merge pull request #1012 from DirectXMan12/bug/fix-prometheus-metrics-filtering

Fix Prometheus Metrics Filtering

collector/config/sample_config_prometheus_filtered.json

@@ -0,0 +1,8 @@
+{
+        "endpoint" : "http://localhost:8080/metrics",
+	"polling_frequency" : 10, 
+        "metrics_config" : [
+            "go_goroutines"
+        ]
+}
+

collector/prometheus_collector.go

@@ -35,6 +35,9 @@ type PrometheusCollector struct {
 
 	//holds information extracted from the config file for a collector
 	configFile Prometheus
+
+	// the metrics to gather (uses a map as a set)
+	metricsSet map[string]bool
 }
 
 //Returns a new collector using the information extracted from the configfile
@@ -54,11 +57,20 @@ func NewPrometheusCollector(collectorName string, configFile []byte) (*Prometheu
 		minPollingFrequency = minSupportedFrequency
 	}
 
+	var metricsSet map[string]bool
+	if len(configInJSON.MetricsConfig) > 0 {
+		metricsSet = make(map[string]bool, len(configInJSON.MetricsConfig))
+		for _, name := range configInJSON.MetricsConfig {
+			metricsSet[name] = true
+		}
+	}
+
 	//TODO : Add checks for validity of config file (eg : Accurate JSON fields)
 	return &PrometheusCollector{
 		name:             collectorName,
 		pollingFrequency: minPollingFrequency,
 		configFile:       configInJSON,
+		metricsSet:       metricsSet,
 	}, nil
 }
 
@@ -100,8 +112,12 @@ func (collector *PrometheusCollector) GetSpec() []v1.MetricSpec {
 			if stopIndex == -1 {
 				stopIndex = strings.Index(lines[i+2], " ")
 			}
+			name := strings.TrimSpace(lines[i+2][0:stopIndex])
+			if _, ok := collector.metricsSet[name]; collector.metricsSet != nil && !ok {
+				continue
+			}
 			spec := v1.MetricSpec{
-				Name:   strings.TrimSpace(lines[i+2][0:stopIndex]),
+				Name:   name,
 				Type:   v1.MetricType(getMetricData(lines[i+1])),
 				Format: "float",
 				Units:  getMetricData(lines[i]),
@@ -145,6 +161,9 @@ func (collector *PrometheusCollector) Collect(metrics map[string][]v1.MetricVal)
 			}
 
 			metName := strings.TrimSpace(line[0:startLabelIndex])
+			if _, ok := collector.metricsSet[metName]; collector.metricsSet != nil && !ok {
+				continue
+			}
 
 			if startLabelIndex+1 <= spaceIndex-1 {
 				metLabel = strings.TrimSpace(line[(startLabelIndex + 1):(spaceIndex - 1)])

collector/prometheus_collector_test.go

@@ -63,3 +63,38 @@ func TestPrometheus(t *testing.T) {
 	goRoutines := metrics["go_goroutines"]
 	assert.Equal(goRoutines[0].FloatValue, 16)
 }
+
+func TestPrometheusFiltersMetrics(t *testing.T) {
+	assert := assert.New(t)
+
+	//Create a prometheus collector using the config file 'sample_config_prometheus_filtered.json'
+	configFile, err := ioutil.ReadFile("config/sample_config_prometheus_filtered.json")
+	collector, err := NewPrometheusCollector("Prometheus", configFile)
+	assert.NoError(err)
+	assert.Equal(collector.name, "Prometheus")
+	assert.Equal(collector.configFile.Endpoint, "http://localhost:8080/metrics")
+
+	tempServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		text := "# HELP go_gc_duration_seconds A summary of the GC invocation durations.\n"
+		text += "# TYPE go_gc_duration_seconds summary\n"
+		text += "go_gc_duration_seconds{quantile=\"0\"} 5.8348000000000004e-05\n"
+		text += "go_gc_duration_seconds{quantile=\"1\"} 0.000499764\n"
+		text += "# HELP go_goroutines Number of goroutines that currently exist.\n"
+		text += "# TYPE go_goroutines gauge\n"
+		text += "go_goroutines 16"
+		fmt.Fprintln(w, text)
+	}))
+
+	defer tempServer.Close()
+
+	collector.configFile.Endpoint = tempServer.URL
+	metrics := map[string][]v1.MetricVal{}
+	_, metrics, errMetric := collector.Collect(metrics)
+
+	assert.NoError(errMetric)
+	assert.Len(metrics, 1)
+
+	goRoutines := metrics["go_goroutines"]
+	assert.Equal(goRoutines[0].FloatValue, 16)
+}

docs/application_metrics.md

@@ -61,19 +61,18 @@ Another sample config that collects only selected metrics:
 {
   "endpoint" : "http://localhost:8000/metrics",
   "metrics_config" : [
-    {
+    "scheduler_binding_latency",
-      "scheduler_binding_latency",
+    "scheduler_e2e_scheduling_latency",
-      "scheduler_e2e_scheduling_latency",
+    "scheduling_algorithm_latency"
-      "scheduling_algorithm_latency"
-    }
   ]
 }
 ```
 
 ## Passing the configuration to cAdvisor
 
-cAdvisor can discover any configurations for a container using Docker labels. Any label starting with ```io.cadvisor.metric``` is parsed as a cadvisor application-metric label.
+cAdvisor can discover any configurations for a container using Docker container labels. Any label starting with ```io.cadvisor.metric``` is parsed as a cadvisor application-metric label.
-cAdvisor uses the value as an indicator of where the configuration can be found.
+cAdvisor uses the value as an indicator of where the configuration can be found.  Labels of the form ```io.cadvisor.metric.prometheus-xyz``` indicate that the configuration points to a
+Prometheus metrics endpoint.
 
 The configuration file can either be part of the container image or can be added on at runtime with a volume. This makes sure that there is no connection between the host where the container is running and the application metrics configuration. A container is self-contained for its metric information.
 
@@ -88,6 +87,9 @@ Dockerfile (or runtime):
 
 cAdvisor will then reach into the container image at runtime, process the config, and start collecting and exposing application metrics.
 
+Note that cAdvisor specifically looks at the container labels to extract this information.  In Docker 1.8, containers don't inherit labels
+from their images, and thus you must specify the label at runtime.
+
 ## API access to application-specific metrics
 
 A new endpoint is added for collecting application-specific metrics for a particular container: