Merge pull request #2262 from sanek9/master

Append application metrics to /metrics so that prometheus can read #1616

collector/prometheus_collector.go

@@ -184,6 +184,17 @@ func (s byName) Less(i, j int) bool {
 	return s.prometheusLabels[i].GetName() < s.prometheusLabels[j].GetName()
 }
 
+func prometheusLabelSetToCadvisorLabels(promLabels model.Metric) map[string]string {
+	labels := make(map[string]string)
+	for k, v := range promLabels {
+		if string(k) == "__name__" {
+			continue
+		}
+		labels[string(k)] = string(v)
+	}
+	return labels
+}
+
 func prometheusLabelSetToCadvisorLabel(promLabels model.Metric) string {
 	labels := labelSetToLabelPairs(promLabels)
 	sort.Sort(byName{labels})
@@ -247,11 +258,13 @@ func (collector *PrometheusCollector) Collect(metrics map[string][]v1.MetricVal)
 			// TODO Handle multiple labels nicer. Prometheus metrics can have multiple
 			// labels, cadvisor only accepts a single string for the metric label.
 			label := prometheusLabelSetToCadvisorLabel(sample.Metric)
+			labels := prometheusLabelSetToCadvisorLabels(sample.Metric)
 
 			metric := v1.MetricVal{
 				FloatValue: float64(sample.Value),
 				Timestamp:  sample.Timestamp.Time(),
 				Label:      label,
+				Labels:     labels,
 			}
 			newMetrics[metName] = append(newMetrics[metName], metric)
 			if len(newMetrics) > collector.metricCountLimit {

info/v1/metric.go

@@ -69,6 +69,7 @@ type MetricValBasic struct {
 type MetricVal struct {
 	// Label associated with a metric
 	Label  string            `json:"label,omitempty"`
+	Labels map[string]string `json:"labels,omitempty"`
 
 	// Time at which the metric was queried
 	Timestamp time.Time `json:"timestamp"`

metrics/prometheus.go

@@ -1674,7 +1674,24 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric)
 				)
 			}
 		}
+		if c.includedMetrics.Has(container.AppMetrics) {
+			for metricLabel, v := range stats.CustomMetrics {
+				for _, metric := range v {
+					clabels := make([]string, len(rawLabels), len(rawLabels)+len(metric.Labels))
+					cvalues := make([]string, len(rawLabels), len(rawLabels)+len(metric.Labels))
+					copy(clabels, labels)
+					copy(cvalues, values)
+					for label, value := range metric.Labels {
+						clabels = append(clabels, sanitizeLabelName("app_"+label))
+						cvalues = append(cvalues, value)
 					}
+					desc := prometheus.NewDesc(metricLabel, "Custom application metric.", clabels, nil)
+					ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(metric.FloatValue), cvalues...)
+				}
+			}
+		}
+	}
+
 }
 
 func (c *PrometheusCollector) collectVersionInfo(ch chan<- prometheus.Metric) {

metrics/prometheus_test.go

@@ -62,6 +62,7 @@ var allMetrics = container.MetricSet{
 	container.NetworkAdvancedTcpUsageMetrics: struct{}{},
 	container.NetworkUdpUsageMetrics:         struct{}{},
 	container.ProcessMetrics:                 struct{}{},
+	container.AppMetrics:                     struct{}{},
 }
 
 func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.ContainerInfoRequest) ([]*info.ContainerInfo, error) {
@@ -377,6 +378,38 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container
 						NrUninterruptible: 53,
 						NrIoWait:          54,
 					},
+					CustomMetrics: map[string][]info.MetricVal{
+						"container_custom_app_metric_1": {
+							{
+								FloatValue: float64(1.1),
+								Timestamp:  time.Now(),
+								Label:      "testlabel_1_1_1",
+								Labels:     map[string]string{"test_label": "1_1", "test_label_2": "2_1"},
+							},
+							{
+								FloatValue: float64(1.2),
+								Timestamp:  time.Now(),
+								Label:      "testlabel_1_1_2",
+								Labels:     map[string]string{"test_label": "1_2", "test_label_2": "2_2"},
+							},
+						},
+						"container_custom_app_metric_2": {
+							{
+								FloatValue: float64(2),
+								Timestamp:  time.Now(),
+								Label:      "testlabel2",
+								Labels:     map[string]string{"test_label": "test_value"},
+							},
+						},
+						"container_custom_app_metric_3": {
+							{
+								FloatValue: float64(3),
+								Timestamp:  time.Now(),
+								Label:      "testlabel3",
+								Labels:     map[string]string{"test_label": "test_value"},
+							},
+						},
+					},
 				},
 			},
 		},

metrics/testdata/prometheus_metrics

@@ -46,6 +46,16 @@ container_cpu_usage_seconds_total{container_env_foo_env="prod",container_label_f
 # HELP container_cpu_user_seconds_total Cumulative user cpu time consumed in seconds.
 # TYPE container_cpu_user_seconds_total counter
 container_cpu_user_seconds_total{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 6e-09 1395066363000
+# HELP container_custom_app_metric_1 Custom application metric.
+# TYPE container_custom_app_metric_1 gauge
+container_custom_app_metric_1{app_test_label="1_1",app_test_label_2="2_1",container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.1
+container_custom_app_metric_1{app_test_label="1_2",app_test_label_2="2_2",container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.2
+# HELP container_custom_app_metric_2 Custom application metric.
+# TYPE container_custom_app_metric_2 gauge
+container_custom_app_metric_2{app_test_label="test_value",container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2
+# HELP container_custom_app_metric_3 Custom application metric.
+# TYPE container_custom_app_metric_3 gauge
+container_custom_app_metric_3{app_test_label="test_value",container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3
 # HELP container_file_descriptors Number of open file descriptors for the container.
 # TYPE container_file_descriptors gauge
 container_file_descriptors{container_env_foo_env="prod",container_label_foo_label="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5 1395066363000