From 923dbc58c14b6b9438ae88525ec559b87e7d21f3 Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Fri, 23 Sep 2016 12:55:26 +0100 Subject: [PATCH] Convert Prometheus labels into single consistent string label for cadvisor --- collector/prometheus_collector.go | 60 +++++++++++++++++++++++--- collector/prometheus_collector_test.go | 20 ++++++++- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/collector/prometheus_collector.go b/collector/prometheus_collector.go index a9a57259..7f4102f6 100644 --- a/collector/prometheus_collector.go +++ b/collector/prometheus_collector.go @@ -15,13 +15,15 @@ package collector import ( + "bytes" "encoding/json" "fmt" "io" "net/http" + "sort" "time" - dto "github.com/prometheus/client_model/go" + rawmodel "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/prometheus/common/model" @@ -118,7 +120,7 @@ func (collector *PrometheusCollector) GetSpec() []v1.MetricSpec { var specs []v1.MetricSpec for { - d := dto.MetricFamily{} + d := rawmodel.MetricFamily{} if err = dec.Decode(&d); err != nil { break } @@ -126,9 +128,11 @@ func (collector *PrometheusCollector) GetSpec() []v1.MetricSpec { if len(name) == 0 { continue } + // If metrics to collect is specified, skip any metrics not in the list to collect. if _, ok := collector.metricsSet[name]; collector.metricsSet != nil && !ok { continue } + spec := v1.MetricSpec{ Name: name, Type: metricType(d.GetType()), @@ -146,16 +150,55 @@ func (collector *PrometheusCollector) GetSpec() []v1.MetricSpec { // metricType converts Prometheus metric type to cadvisor metric type. // If there is no mapping then just return the name of the Prometheus metric type. -func metricType(t dto.MetricType) v1.MetricType { +func metricType(t rawmodel.MetricType) v1.MetricType { switch t { - case dto.MetricType_COUNTER: + case rawmodel.MetricType_COUNTER: return v1.MetricCumulative - case dto.MetricType_GAUGE: + case rawmodel.MetricType_GAUGE: return v1.MetricGauge default: return v1.MetricType(t.String()) } +} +type prometheusLabels []*rawmodel.LabelPair + +func labelSetToLabelPairs(labels model.Metric) prometheusLabels { + var promLabels prometheusLabels + for k, v := range labels { + name := string(k) + value := string(v) + promLabels = append(promLabels, &rawmodel.LabelPair{Name: &name, Value: &value}) + } + return promLabels +} + +func (s prometheusLabels) Len() int { return len(s) } +func (s prometheusLabels) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// ByName implements sort.Interface by providing Less and using the Len and +// Swap methods of the embedded PrometheusLabels value. +type byName struct{ prometheusLabels } + +func (s byName) Less(i, j int) bool { + return s.prometheusLabels[i].GetName() < s.prometheusLabels[j].GetName() +} + +func prometheusLabelSetToCadvisorLabel(promLabels model.Metric) string { + labels := labelSetToLabelPairs(promLabels) + sort.Sort(byName{labels}) + var b bytes.Buffer + + for i, l := range labels { + if i > 0 { + b.WriteString("\xff") + } + b.WriteString(l.GetName()) + b.WriteString("=") + b.WriteString(l.GetValue()) + } + + return string(b.Bytes()) } //Returns collected metrics and the next collection time of the collector @@ -182,6 +225,8 @@ func (collector *PrometheusCollector) Collect(metrics map[string][]v1.MetricVal) } var ( + // 50 is chosen as a reasonable guesstimate at a number of metrics we can + // expect from virtually any endpoint to try to save allocations. decSamples = make(model.Vector, 0, 50) newMetrics = make(map[string][]v1.MetricVal) ) @@ -195,13 +240,18 @@ func (collector *PrometheusCollector) Collect(metrics map[string][]v1.MetricVal) if len(metName) == 0 { continue } + // If metrics to collect is specified, skip any metrics not in the list to collect. if _, ok := collector.metricsSet[metName]; collector.metricsSet != nil && !ok { continue } + // TODO Handle multiple labels nicer. Prometheus metrics can have multiple + // labels, cadvisor only accepts a single string for the metric label. + label := prometheusLabelSetToCadvisorLabel(sample.Metric) metric := v1.MetricVal{ FloatValue: float64(sample.Value), Timestamp: sample.Timestamp.Time(), + Label: label, } newMetrics[metName] = append(newMetrics[metName], metric) if len(newMetrics) > collector.metricCountLimit { diff --git a/collector/prometheus_collector_test.go b/collector/prometheus_collector_test.go index 4285a4dc..67926fbf 100644 --- a/collector/prometheus_collector_test.go +++ b/collector/prometheus_collector_test.go @@ -55,6 +55,9 @@ go_goroutines 16 # HELP metric_with_spaces_in_label A metric with spaces in a label. # TYPE metric_with_spaces_in_label gauge metric_with_spaces_in_label{name="Network Agent"} 72 +# HELP metric_with_multiple_labels A metric with multiple labels. +# TYPE metric_with_multiple_labels gauge +metric_with_multiple_labels{label1="One", label2="Two", label3="Three"} 81 ` fmt.Fprintln(w, text) })) @@ -65,7 +68,7 @@ metric_with_spaces_in_label{name="Network Agent"} 72 var spec []v1.MetricSpec require.NotPanics(t, func() { spec = collector.GetSpec() }) - assert.Len(spec, 3) + assert.Len(spec, 4) specNames := make(map[string]struct{}, 3) for _, s := range spec { specNames[s.Name] = struct{}{} @@ -74,6 +77,7 @@ metric_with_spaces_in_label{name="Network Agent"} 72 "go_gc_duration_seconds": {}, "go_goroutines": {}, "metric_with_spaces_in_label": {}, + "metric_with_multiple_labels": {}, } assert.Equal(expectedSpecNames, specNames) @@ -84,13 +88,27 @@ metric_with_spaces_in_label{name="Network Agent"} 72 go_gc_duration := metrics["go_gc_duration_seconds"] assert.Equal(5.8348000000000004e-05, go_gc_duration[0].FloatValue) + assert.Equal("__name__=go_gc_duration_seconds\xffquantile=0", go_gc_duration[0].Label) assert.Equal(0.000499764, go_gc_duration[1].FloatValue) + assert.Equal("__name__=go_gc_duration_seconds\xffquantile=1", go_gc_duration[1].Label) + go_gc_duration_sum := metrics["go_gc_duration_seconds_sum"] + assert.Equal(1.7560473e+07, go_gc_duration_sum[0].FloatValue) + assert.Equal("__name__=go_gc_duration_seconds_sum", go_gc_duration_sum[0].Label) + go_gc_duration_count := metrics["go_gc_duration_seconds_count"] + assert.Equal(2693, go_gc_duration_count[0].FloatValue) + assert.Equal("__name__=go_gc_duration_seconds_count", go_gc_duration_count[0].Label) goRoutines := metrics["go_goroutines"] assert.Equal(16, goRoutines[0].FloatValue) + assert.Equal("__name__=go_goroutines", goRoutines[0].Label) metricWithSpaces := metrics["metric_with_spaces_in_label"] assert.Equal(72, metricWithSpaces[0].FloatValue) + assert.Equal("__name__=metric_with_spaces_in_label\xffname=Network Agent", metricWithSpaces[0].Label) + + metricWithMultipleLabels := metrics["metric_with_multiple_labels"] + assert.Equal(81, metricWithMultipleLabels[0].FloatValue) + assert.Equal("__name__=metric_with_multiple_labels\xfflabel1=One\xfflabel2=Two\xfflabel3=Three", metricWithMultipleLabels[0].Label) } func TestPrometheusEndpointConfig(t *testing.T) {