From e76096d4f6387397a76cc34416ff75d8479053ef Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Sat, 20 Aug 2016 16:47:53 -0400 Subject: [PATCH] Provide option to control Prometheus labels This change generalizes the existing ContainerNameToLabelsFunc to allow the user to fully control all labels attached to exported Prometheus metrics. The existing behavior is available as DefaultContainerLabelsFunc and is used if no custom function is provided. This will allow Kubernetes to filter out its internal Docker labels. --- http/handlers.go | 7 +- metrics/prometheus.go | 131 ++++++++++++++++++++----------------- metrics/prometheus_test.go | 18 ++--- 3 files changed, 86 insertions(+), 70 deletions(-) diff --git a/http/handlers.go b/http/handlers.go index 48e088da..420077d7 100644 --- a/http/handlers.go +++ b/http/handlers.go @@ -89,8 +89,11 @@ func RegisterHandlers(mux httpmux.Mux, containerManager manager.Manager, httpAut return nil } -func RegisterPrometheusHandler(mux httpmux.Mux, containerManager manager.Manager, prometheusEndpoint string, containerNameToLabelsFunc metrics.ContainerNameToLabelsFunc) { - collector := metrics.NewPrometheusCollector(containerManager, containerNameToLabelsFunc) +// RegisterPrometheusHandler creates a new PrometheusCollector, registers it +// on the global registry and configures the provided HTTP mux to handle the +// given Prometheus endpoint. +func RegisterPrometheusHandler(mux httpmux.Mux, containerManager manager.Manager, prometheusEndpoint string, f metrics.ContainerLabelsFunc) { + collector := metrics.NewPrometheusCollector(containerManager, f) prometheus.MustRegister(collector) mux.Handle(prometheusEndpoint, prometheus.Handler()) } diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 0e7e8fba..7d738599 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -25,13 +25,14 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -// This will usually be manager.Manager, but can be swapped out for testing. +// infoProvider will usually be manager.Manager, but can be swapped out for testing. type infoProvider interface { - // Get information about all subcontainers of the specified container (includes self). + // SubcontainersInfo provides information about all subcontainers of the + // specified container including itself. SubcontainersInfo(containerName string, query *info.ContainerInfoRequest) ([]*info.ContainerInfo, error) - // Get information about the version. + // GetVersionInfo provides information about the version. GetVersionInfo() (*info.VersionInfo, error) - // Get information about the machine. + // GetMachineInfo provides information about the machine. GetMachineInfo() (*info.MachineInfo, error) } @@ -56,8 +57,8 @@ func fsValues(fsStats []info.FsStats, valueFn func(*info.FsStats) float64) metri return values } -// A containerMetric describes a multi-dimensional metric used for exposing -// a certain type of container statistic. +// containerMetric describes a multi-dimensional metric used for exposing a +// certain type of container statistic. type containerMetric struct { name string help string @@ -71,21 +72,29 @@ func (cm *containerMetric) desc(baseLabels []string) *prometheus.Desc { return prometheus.NewDesc(cm.name, cm.help, append(baseLabels, cm.extraLabels...), nil) } -type ContainerNameToLabelsFunc func(containerName string) map[string]string +// ContainerLabelsFunc defines all base labels and their values attached to +// each metric exported by cAdvisor. +type ContainerLabelsFunc func(*info.ContainerInfo) map[string]string // PrometheusCollector implements prometheus.Collector. type PrometheusCollector struct { - infoProvider infoProvider - errors prometheus.Gauge - containerMetrics []containerMetric - containerNameToLabels ContainerNameToLabelsFunc + infoProvider infoProvider + errors prometheus.Gauge + containerMetrics []containerMetric + containerLabelsFunc ContainerLabelsFunc } -// NewPrometheusCollector returns a new PrometheusCollector. -func NewPrometheusCollector(infoProvider infoProvider, f ContainerNameToLabelsFunc) *PrometheusCollector { +// NewPrometheusCollector returns a new PrometheusCollector. The passed +// ContainerLabelsFunc specifies which base labels will be attached to all +// exported metrics. If left to nil, the DefaultContainerLabels function +// will be used instead. +func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc) *PrometheusCollector { + if f == nil { + f = DefaultContainerLabels + } c := &PrometheusCollector{ - infoProvider: infoProvider, - containerNameToLabels: f, + infoProvider: i, + containerLabelsFunc: f, errors: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "container", Name: "scrape_error", @@ -533,10 +542,38 @@ func (c *PrometheusCollector) Collect(ch chan<- prometheus.Metric) { } const ( - containerLabelPrefix = "container_label_" - containerEnvPrefix = "container_env_" + // ContainerLabelPrefix is the prefix added to all container labels. + ContainerLabelPrefix = "container_label_" + // ContainerEnvPrefix is the prefix added to all env variable labels. + ContainerEnvPrefix = "container_env_" + // LabelID is the name of the id label. + LabelID = "id" + // LabelName is the name of the name label. + LabelName = "name" + // LabelImage is the name of the image label. + LabelImage = "image" ) +// DefaultContainerLabels implements ContainerLabelsFunc. It exports the +// container name, first alias, image name as well as all its env and label +// values. +func DefaultContainerLabels(container *info.ContainerInfo) map[string]string { + set := map[string]string{LabelID: container.Name} + if len(container.Aliases) > 0 { + set[LabelName] = container.Aliases[0] + } + if image := container.Spec.Image; len(image) > 0 { + set[LabelImage] = image + } + for k, v := range container.Spec.Labels { + set[ContainerLabelPrefix+k] = v + } + for k, v := range container.Spec.Envs { + set[ContainerEnvPrefix+k] = v + } + return set +} + func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) { containers, err := c.infoProvider.SubcontainersInfo("/", &info.ContainerInfoRequest{NumStats: 1}) if err != nil { @@ -545,56 +582,32 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) return } for _, container := range containers { - baseLabels := []string{"id"} - id := container.Name - name := id - if len(container.Aliases) > 0 { - name = container.Aliases[0] - baseLabels = append(baseLabels, "name") - } - image := container.Spec.Image - if len(image) > 0 { - baseLabels = append(baseLabels, "image") - } - baseLabelValues := []string{id, name, image}[:len(baseLabels)] - - if c.containerNameToLabels != nil { - newLabels := c.containerNameToLabels(name) - for k, v := range newLabels { - baseLabels = append(baseLabels, sanitizeLabelName(k)) - baseLabelValues = append(baseLabelValues, v) - } - } - - for k, v := range container.Spec.Labels { - baseLabels = append(baseLabels, sanitizeLabelName(containerLabelPrefix+k)) - baseLabelValues = append(baseLabelValues, v) - } - for k, v := range container.Spec.Envs { - baseLabels = append(baseLabels, sanitizeLabelName(containerEnvPrefix+k)) - baseLabelValues = append(baseLabelValues, v) + labels, values := []string{}, []string{} + for l, v := range c.containerLabelsFunc(container) { + labels = append(labels, sanitizeLabelName(l)) + values = append(values, v) } // Container spec - desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), baseLabelValues...) + desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), values...) if container.Spec.HasCpu { - desc = prometheus.NewDesc("container_spec_cpu_period", "CPU period of the container.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Period), baseLabelValues...) + desc = prometheus.NewDesc("container_spec_cpu_period", "CPU period of the container.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Period), values...) if container.Spec.Cpu.Quota != 0 { - desc = prometheus.NewDesc("container_spec_cpu_quota", "CPU quota of the container.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Quota), baseLabelValues...) + desc = prometheus.NewDesc("container_spec_cpu_quota", "CPU quota of the container.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Quota), values...) } - desc := prometheus.NewDesc("container_spec_cpu_shares", "CPU share of the container.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Limit), baseLabelValues...) + desc := prometheus.NewDesc("container_spec_cpu_shares", "CPU share of the container.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Limit), values...) } if container.Spec.HasMemory { - desc := prometheus.NewDesc("container_spec_memory_limit_bytes", "Memory limit for the container.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Limit), baseLabelValues...) - desc = prometheus.NewDesc("container_spec_memory_swap_limit_bytes", "Memory swap limit for the container.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.SwapLimit), baseLabelValues...) + desc := prometheus.NewDesc("container_spec_memory_limit_bytes", "Memory limit for the container.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Limit), values...) + desc = prometheus.NewDesc("container_spec_memory_swap_limit_bytes", "Memory swap limit for the container.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.SwapLimit), values...) } // Now for the actual metrics @@ -603,9 +616,9 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) if cm.condition != nil && !cm.condition(container.Spec) { continue } - desc := cm.desc(baseLabels) + desc := cm.desc(labels) for _, metricValue := range cm.getValues(stats) { - ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append(baseLabelValues, metricValue.labels...)...) + ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append(values, metricValue.labels...)...) } } } diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go index 8c54097e..d62a64ec 100644 --- a/metrics/prometheus_test.go +++ b/metrics/prometheus_test.go @@ -180,10 +180,10 @@ var ( ) func TestPrometheusCollector(t *testing.T) { - c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(name string) map[string]string { - return map[string]string{ - "zone.name": "hello", - } + c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(container *info.ContainerInfo) map[string]string { + s := DefaultContainerLabels(container) + s["zone.name"] = "hello" + return s }) prometheus.MustRegister(c) defer prometheus.Unregister(c) @@ -212,7 +212,7 @@ func testPrometheusCollector(t *testing.T, c *PrometheusCollector, metricsFile s continue } if want != gotLines[i] { - t.Fatalf("want %s, got %s", want, gotLines[i]) + t.Fatalf("unexpected metric line\nwant: %s\nhave: %s", want, gotLines[i]) } } } @@ -250,10 +250,10 @@ func TestPrometheusCollector_scrapeFailure(t *testing.T) { shouldFail: true, } - c := NewPrometheusCollector(provider, func(name string) map[string]string { - return map[string]string{ - "zone.name": "hello", - } + c := NewPrometheusCollector(provider, func(container *info.ContainerInfo) map[string]string { + s := DefaultContainerLabels(container) + s["zone.name"] = "hello" + return s }) prometheus.MustRegister(c) defer prometheus.Unregister(c)