Convert Prometheus labels into single consistent string label for cadvisor

collector/prometheus_collector.go

@@ -15,13 +15,15 @@
 package collector
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"sort"
 	"time"
 
-	dto "github.com/prometheus/client_model/go"
+	rawmodel "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
 	"github.com/prometheus/common/model"
 
@@ -118,7 +120,7 @@ func (collector *PrometheusCollector) GetSpec() []v1.MetricSpec {
 	var specs []v1.MetricSpec
 
 	for {
-		d := dto.MetricFamily{}
+		d := rawmodel.MetricFamily{}
 		if err = dec.Decode(&d); err != nil {
 			break
 		}
@@ -126,9 +128,11 @@ func (collector *PrometheusCollector) GetSpec() []v1.MetricSpec {
 		if len(name) == 0 {
 			continue
 		}
+		// If metrics to collect is specified, skip any metrics not in the list to collect.
 		if _, ok := collector.metricsSet[name]; collector.metricsSet != nil && !ok {
 			continue
 		}
+
 		spec := v1.MetricSpec{
 			Name:   name,
 			Type:   metricType(d.GetType()),
@@ -146,16 +150,55 @@ func (collector *PrometheusCollector) GetSpec() []v1.MetricSpec {
 
 // metricType converts Prometheus metric type to cadvisor metric type.
 // If there is no mapping then just return the name of the Prometheus metric type.
-func metricType(t dto.MetricType) v1.MetricType {
+func metricType(t rawmodel.MetricType) v1.MetricType {
 	switch t {
-	case dto.MetricType_COUNTER:
+	case rawmodel.MetricType_COUNTER:
 		return v1.MetricCumulative
-	case dto.MetricType_GAUGE:
+	case rawmodel.MetricType_GAUGE:
 		return v1.MetricGauge
 	default:
 		return v1.MetricType(t.String())
 	}
+}
 
+type prometheusLabels []*rawmodel.LabelPair
+
+func labelSetToLabelPairs(labels model.Metric) prometheusLabels {
+	var promLabels prometheusLabels
+	for k, v := range labels {
+		name := string(k)
+		value := string(v)
+		promLabels = append(promLabels, &rawmodel.LabelPair{Name: &name, Value: &value})
+	}
+	return promLabels
+}
+
+func (s prometheusLabels) Len() int      { return len(s) }
+func (s prometheusLabels) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+// ByName implements sort.Interface by providing Less and using the Len and
+// Swap methods of the embedded PrometheusLabels value.
+type byName struct{ prometheusLabels }
+
+func (s byName) Less(i, j int) bool {
+	return s.prometheusLabels[i].GetName() < s.prometheusLabels[j].GetName()
+}
+
+func prometheusLabelSetToCadvisorLabel(promLabels model.Metric) string {
+	labels := labelSetToLabelPairs(promLabels)
+	sort.Sort(byName{labels})
+	var b bytes.Buffer
+
+	for i, l := range labels {
+		if i > 0 {
+			b.WriteString("\xff")
+		}
+		b.WriteString(l.GetName())
+		b.WriteString("=")
+		b.WriteString(l.GetValue())
+	}
+
+	return string(b.Bytes())
 }
 
 //Returns collected metrics and the next collection time of the collector
@@ -182,6 +225,8 @@ func (collector *PrometheusCollector) Collect(metrics map[string][]v1.MetricVal)
 	}
 
 	var (
+		// 50 is chosen as a reasonable guesstimate at a number of metrics we can
+		// expect from virtually any endpoint to try to save allocations.
 		decSamples = make(model.Vector, 0, 50)
 		newMetrics = make(map[string][]v1.MetricVal)
 	)
@@ -195,13 +240,18 @@ func (collector *PrometheusCollector) Collect(metrics map[string][]v1.MetricVal)
 			if len(metName) == 0 {
 				continue
 			}
+			// If metrics to collect is specified, skip any metrics not in the list to collect.
 			if _, ok := collector.metricsSet[metName]; collector.metricsSet != nil && !ok {
 				continue
 			}
+			// TODO Handle multiple labels nicer. Prometheus metrics can have multiple
+			// labels, cadvisor only accepts a single string for the metric label.
+			label := prometheusLabelSetToCadvisorLabel(sample.Metric)
 
 			metric := v1.MetricVal{
 				FloatValue: float64(sample.Value),
 				Timestamp:  sample.Timestamp.Time(),
+				Label:      label,
 			}
 			newMetrics[metName] = append(newMetrics[metName], metric)
 			if len(newMetrics) > collector.metricCountLimit {

collector/prometheus_collector_test.go

@@ -55,6 +55,9 @@ go_goroutines 16
 # HELP metric_with_spaces_in_label A metric with spaces in a label.
 # TYPE metric_with_spaces_in_label gauge
 metric_with_spaces_in_label{name="Network Agent"} 72
+# HELP metric_with_multiple_labels A metric with multiple labels.
+# TYPE metric_with_multiple_labels gauge
+metric_with_multiple_labels{label1="One", label2="Two", label3="Three"} 81
 `
 		fmt.Fprintln(w, text)
 	}))
@@ -65,7 +68,7 @@ metric_with_spaces_in_label{name="Network Agent"} 72
 
 	var spec []v1.MetricSpec
 	require.NotPanics(t, func() { spec = collector.GetSpec() })
-	assert.Len(spec, 3)
+	assert.Len(spec, 4)
 	specNames := make(map[string]struct{}, 3)
 	for _, s := range spec {
 		specNames[s.Name] = struct{}{}
@@ -74,6 +77,7 @@ metric_with_spaces_in_label{name="Network Agent"} 72
 		"go_gc_duration_seconds":      {},
 		"go_goroutines":               {},
 		"metric_with_spaces_in_label": {},
+		"metric_with_multiple_labels": {},
 	}
 	assert.Equal(expectedSpecNames, specNames)
 
@@ -84,13 +88,27 @@ metric_with_spaces_in_label{name="Network Agent"} 72
 
 	go_gc_duration := metrics["go_gc_duration_seconds"]
 	assert.Equal(5.8348000000000004e-05, go_gc_duration[0].FloatValue)
+	assert.Equal("__name__=go_gc_duration_seconds\xffquantile=0", go_gc_duration[0].Label)
 	assert.Equal(0.000499764, go_gc_duration[1].FloatValue)
+	assert.Equal("__name__=go_gc_duration_seconds\xffquantile=1", go_gc_duration[1].Label)
+	go_gc_duration_sum := metrics["go_gc_duration_seconds_sum"]
+	assert.Equal(1.7560473e+07, go_gc_duration_sum[0].FloatValue)
+	assert.Equal("__name__=go_gc_duration_seconds_sum", go_gc_duration_sum[0].Label)
+	go_gc_duration_count := metrics["go_gc_duration_seconds_count"]
+	assert.Equal(2693, go_gc_duration_count[0].FloatValue)
+	assert.Equal("__name__=go_gc_duration_seconds_count", go_gc_duration_count[0].Label)
 
 	goRoutines := metrics["go_goroutines"]
 	assert.Equal(16, goRoutines[0].FloatValue)
+	assert.Equal("__name__=go_goroutines", goRoutines[0].Label)
 
 	metricWithSpaces := metrics["metric_with_spaces_in_label"]
 	assert.Equal(72, metricWithSpaces[0].FloatValue)
+	assert.Equal("__name__=metric_with_spaces_in_label\xffname=Network Agent", metricWithSpaces[0].Label)
+
+	metricWithMultipleLabels := metrics["metric_with_multiple_labels"]
+	assert.Equal(81, metricWithMultipleLabels[0].FloatValue)
+	assert.Equal("__name__=metric_with_multiple_labels\xfflabel1=One\xfflabel2=Two\xfflabel3=Three", metricWithMultipleLabels[0].Label)
 }
 
 func TestPrometheusEndpointConfig(t *testing.T) {