From d11dd29ecda8ded80444b3562e19b9f491a47bcf Mon Sep 17 00:00:00 2001 From: Solly Ross Date: Wed, 9 Dec 2015 18:30:41 -0500 Subject: [PATCH 1/2] Make Prometheus Collector Filter Custom Metrics Previously, the Prometheus collector ignored the `MetricsConfig` field of the custom metrics specification, simply storing all exposed metrics. Now, if the `MetricsConfig` field contains any metric names, only those metrics will be stored and exposed. Fixes #1005 --- .../sample_config_prometheus_filtered.json | 8 +++++ collector/prometheus_collector.go | 21 ++++++++++- collector/prometheus_collector_test.go | 35 +++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 collector/config/sample_config_prometheus_filtered.json diff --git a/collector/config/sample_config_prometheus_filtered.json b/collector/config/sample_config_prometheus_filtered.json new file mode 100644 index 00000000..1db3a9d2 --- /dev/null +++ b/collector/config/sample_config_prometheus_filtered.json @@ -0,0 +1,8 @@ +{ + "endpoint" : "http://localhost:8080/metrics", + "polling_frequency" : 10, + "metrics_config" : [ + "go_goroutines" + ] +} + diff --git a/collector/prometheus_collector.go b/collector/prometheus_collector.go index 9253a06d..aa91644c 100644 --- a/collector/prometheus_collector.go +++ b/collector/prometheus_collector.go @@ -35,6 +35,9 @@ type PrometheusCollector struct { //holds information extracted from the config file for a collector configFile Prometheus + + // the metrics to gather (uses a map as a set) + metricsSet map[string]bool } //Returns a new collector using the information extracted from the configfile @@ -54,11 +57,20 @@ func NewPrometheusCollector(collectorName string, configFile []byte) (*Prometheu minPollingFrequency = minSupportedFrequency } + var metricsSet map[string]bool + if len(configInJSON.MetricsConfig) > 0 { + metricsSet = make(map[string]bool, len(configInJSON.MetricsConfig)) + for _, name := range configInJSON.MetricsConfig { + metricsSet[name] = true + } + } + //TODO : Add checks for validity of config file (eg : Accurate JSON fields) return &PrometheusCollector{ name: collectorName, pollingFrequency: minPollingFrequency, configFile: configInJSON, + metricsSet: metricsSet, }, nil } @@ -100,8 +112,12 @@ func (collector *PrometheusCollector) GetSpec() []v1.MetricSpec { if stopIndex == -1 { stopIndex = strings.Index(lines[i+2], " ") } + name := strings.TrimSpace(lines[i+2][0:stopIndex]) + if _, ok := collector.metricsSet[name]; collector.metricsSet != nil && !ok { + continue + } spec := v1.MetricSpec{ - Name: strings.TrimSpace(lines[i+2][0:stopIndex]), + Name: name, Type: v1.MetricType(getMetricData(lines[i+1])), Format: "float", Units: getMetricData(lines[i]), @@ -145,6 +161,9 @@ func (collector *PrometheusCollector) Collect(metrics map[string][]v1.MetricVal) } metName := strings.TrimSpace(line[0:startLabelIndex]) + if _, ok := collector.metricsSet[metName]; collector.metricsSet != nil && !ok { + continue + } if startLabelIndex+1 <= spaceIndex-1 { metLabel = strings.TrimSpace(line[(startLabelIndex + 1):(spaceIndex - 1)]) diff --git a/collector/prometheus_collector_test.go b/collector/prometheus_collector_test.go index 90b425ac..6384a941 100644 --- a/collector/prometheus_collector_test.go +++ b/collector/prometheus_collector_test.go @@ -63,3 +63,38 @@ func TestPrometheus(t *testing.T) { goRoutines := metrics["go_goroutines"] assert.Equal(goRoutines[0].FloatValue, 16) } + +func TestPrometheusFiltersMetrics(t *testing.T) { + assert := assert.New(t) + + //Create a prometheus collector using the config file 'sample_config_prometheus_filtered.json' + configFile, err := ioutil.ReadFile("config/sample_config_prometheus_filtered.json") + collector, err := NewPrometheusCollector("Prometheus", configFile) + assert.NoError(err) + assert.Equal(collector.name, "Prometheus") + assert.Equal(collector.configFile.Endpoint, "http://localhost:8080/metrics") + + tempServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + text := "# HELP go_gc_duration_seconds A summary of the GC invocation durations.\n" + text += "# TYPE go_gc_duration_seconds summary\n" + text += "go_gc_duration_seconds{quantile=\"0\"} 5.8348000000000004e-05\n" + text += "go_gc_duration_seconds{quantile=\"1\"} 0.000499764\n" + text += "# HELP go_goroutines Number of goroutines that currently exist.\n" + text += "# TYPE go_goroutines gauge\n" + text += "go_goroutines 16" + fmt.Fprintln(w, text) + })) + + defer tempServer.Close() + + collector.configFile.Endpoint = tempServer.URL + metrics := map[string][]v1.MetricVal{} + _, metrics, errMetric := collector.Collect(metrics) + + assert.NoError(errMetric) + assert.Len(metrics, 1) + + goRoutines := metrics["go_goroutines"] + assert.Equal(goRoutines[0].FloatValue, 16) +} From 89e656d2d887fcbb452a142e17d840e77f3fd33f Mon Sep 17 00:00:00 2001 From: Solly Ross Date: Wed, 9 Dec 2015 19:40:23 -0500 Subject: [PATCH 2/2] Fix Custom Metrics Documentation The documentation for custom metrics contained an error in the Prometheus metrics example JSON, and was also somewhat unclear as to how the labels worked. --- docs/application_metrics.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/application_metrics.md b/docs/application_metrics.md index a947e55c..bb3b1861 100644 --- a/docs/application_metrics.md +++ b/docs/application_metrics.md @@ -61,19 +61,18 @@ Another sample config that collects only selected metrics: { "endpoint" : "http://localhost:8000/metrics", "metrics_config" : [ - { - "scheduler_binding_latency", - "scheduler_e2e_scheduling_latency", - "scheduling_algorithm_latency" - } + "scheduler_binding_latency", + "scheduler_e2e_scheduling_latency", + "scheduling_algorithm_latency" ] } ``` ## Passing the configuration to cAdvisor -cAdvisor can discover any configurations for a container using Docker labels. Any label starting with ```io.cadvisor.metric``` is parsed as a cadvisor application-metric label. -cAdvisor uses the value as an indicator of where the configuration can be found. +cAdvisor can discover any configurations for a container using Docker container labels. Any label starting with ```io.cadvisor.metric``` is parsed as a cadvisor application-metric label. +cAdvisor uses the value as an indicator of where the configuration can be found. Labels of the form ```io.cadvisor.metric.prometheus-xyz``` indicate that the configuration points to a +Prometheus metrics endpoint. The configuration file can either be part of the container image or can be added on at runtime with a volume. This makes sure that there is no connection between the host where the container is running and the application metrics configuration. A container is self-contained for its metric information. @@ -88,6 +87,9 @@ Dockerfile (or runtime): cAdvisor will then reach into the container image at runtime, process the config, and start collecting and exposing application metrics. +Note that cAdvisor specifically looks at the container labels to extract this information. In Docker 1.8, containers don't inherit labels +from their images, and thus you must specify the label at runtime. + ## API access to application-specific metrics A new endpoint is added for collecting application-specific metrics for a particular container: