From 22c5e624bc6575ea24e04786e41218f55ab250a3 Mon Sep 17 00:00:00 2001 From: Alexander Voitov Date: Mon, 1 Jul 2019 21:51:30 +0300 Subject: [PATCH] export application metrics via /metrics --- collector/prometheus_collector.go | 10 ++++++++++ info/v1/metric.go | 3 ++- metrics/prometheus.go | 31 +++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/collector/prometheus_collector.go b/collector/prometheus_collector.go index efc3f3f8..baad487b 100644 --- a/collector/prometheus_collector.go +++ b/collector/prometheus_collector.go @@ -184,6 +184,14 @@ func (s byName) Less(i, j int) bool { return s.prometheusLabels[i].GetName() < s.prometheusLabels[j].GetName() } +func prometheusLabelSetToCadvisorLabels(promLabels model.Metric) map[string]string { + labels := make(map[string]string) + for k, v := range promLabels { + labels[string(k)] = string(v) + } + return labels +} + func prometheusLabelSetToCadvisorLabel(promLabels model.Metric) string { labels := labelSetToLabelPairs(promLabels) sort.Sort(byName{labels}) @@ -247,11 +255,13 @@ func (collector *PrometheusCollector) Collect(metrics map[string][]v1.MetricVal) // TODO Handle multiple labels nicer. Prometheus metrics can have multiple // labels, cadvisor only accepts a single string for the metric label. label := prometheusLabelSetToCadvisorLabel(sample.Metric) + labels := prometheusLabelSetToCadvisorLabels(sample.Metric) metric := v1.MetricVal{ FloatValue: float64(sample.Value), Timestamp: sample.Timestamp.Time(), Label: label, + Labels: labels, } newMetrics[metName] = append(newMetrics[metName], metric) if len(newMetrics) > collector.metricCountLimit { diff --git a/info/v1/metric.go b/info/v1/metric.go index 30c23ed1..39d1ac22 100644 --- a/info/v1/metric.go +++ b/info/v1/metric.go @@ -68,7 +68,8 @@ type MetricValBasic struct { // An exported metric. type MetricVal struct { // Label associated with a metric - Label string `json:"label,omitempty"` + Label string `json:"label,omitempty"` + Labels map[string]string `json:"labels,omitempty"` // Time at which the metric was queried Timestamp time.Time `json:"timestamp"` diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 2cdb08ab..9b2fbb7f 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -17,6 +17,7 @@ package metrics import ( "fmt" "regexp" + "strings" "time" "github.com/google/cadvisor/container" @@ -1675,6 +1676,36 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) } } } + + if c.includedMetrics.Has(container.AppMetrics) { + for _, container := range containers { + cstats := container.Stats + if len(cstats) > 0 { + last := cstats[len(cstats)-1] + for metricLabel, v := range last.CustomMetrics { + for _, metric := range v { + values := make([]string, 0, len(metric.Labels)+2) + labels := make([]string, 0, len(metric.Labels)+2) + labels = append(labels, "container_name") + values = append(values, container.ContainerReference.Name) + alias := strings.Join(container.ContainerReference.Aliases, ".") + labels = append(labels, "container_alias") + values = append(values, alias) + for label, value := range metric.Labels { + if label == "__name__" { + continue + } + labels = append(labels, sanitizeLabelName(label)) + values = append(values, value) + } + desc := prometheus.NewDesc(metricLabel, "Custom application metric.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(metric.FloatValue), values...) + } + } + + } + } + } } func (c *PrometheusCollector) collectVersionInfo(ch chan<- prometheus.Metric) {