prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Provide option to control Prometheus labels

Browse Source 
This change generalizes the existing ContainerNameToLabelsFunc to allow the user to fully control all labels attached to exported Prometheus metrics. The existing behavior is available as DefaultContainerLabelsFunc and is used if no custom function is provided.

This will allow Kubernetes to filter out its internal Docker labels.
This commit is contained in:
Tobias Schmidt 2016-08-20 16:47:53 -04:00
parent 2ed7198f77
commit e76096d4f6
3 changed files with 86 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
http/handlers.go
View File

@ -89,8 +89,11 @@ func RegisterHandlers(mux httpmux.Mux, containerManager manager.Manager, httpAut
return nil return nil
} }
func RegisterPrometheusHandler(mux httpmux.Mux, containerManager manager.Manager, prometheusEndpoint string, containerNameToLabelsFunc metrics.ContainerNameToLabelsFunc) { // RegisterPrometheusHandler creates a new PrometheusCollector, registers it
collector := metrics.NewPrometheusCollector(containerManager, containerNameToLabelsFunc) // on the global registry and configures the provided HTTP mux to handle the
// given Prometheus endpoint.
func RegisterPrometheusHandler(mux httpmux.Mux, containerManager manager.Manager, prometheusEndpoint string, f metrics.ContainerLabelsFunc) {
collector := metrics.NewPrometheusCollector(containerManager, f)
prometheus.MustRegister(collector) prometheus.MustRegister(collector)
mux.Handle(prometheusEndpoint, prometheus.Handler()) mux.Handle(prometheusEndpoint, prometheus.Handler())
} }

125
metrics/prometheus.go
View File

@ -25,13 +25,14 @@ import (
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
) )
// This will usually be manager.Manager, but can be swapped out for testing. // infoProvider will usually be manager.Manager, but can be swapped out for testing.
type infoProvider interface { type infoProvider interface {
// Get information about all subcontainers of the specified container (includes self). // SubcontainersInfo provides information about all subcontainers of the
// specified container including itself.
SubcontainersInfo(containerName string, query *info.ContainerInfoRequest) ([]*info.ContainerInfo, error) SubcontainersInfo(containerName string, query *info.ContainerInfoRequest) ([]*info.ContainerInfo, error)
// Get information about the version. // GetVersionInfo provides information about the version.
GetVersionInfo() (*info.VersionInfo, error) GetVersionInfo() (*info.VersionInfo, error)
// Get information about the machine. // GetMachineInfo provides information about the machine.
GetMachineInfo() (*info.MachineInfo, error) GetMachineInfo() (*info.MachineInfo, error)
} }
@ -56,8 +57,8 @@ func fsValues(fsStats []info.FsStats, valueFn func(*info.FsStats) float64) metri
return values return values
} }
// A containerMetric describes a multi-dimensional metric used for exposing // containerMetric describes a multi-dimensional metric used for exposing a
// a certain type of container statistic. // certain type of container statistic.
type containerMetric struct { type containerMetric struct {
name string name string
help string help string
@ -71,21 +72,29 @@ func (cm *containerMetric) desc(baseLabels []string) *prometheus.Desc {
return prometheus.NewDesc(cm.name, cm.help, append(baseLabels, cm.extraLabels...), nil) return prometheus.NewDesc(cm.name, cm.help, append(baseLabels, cm.extraLabels...), nil)
} }
type ContainerNameToLabelsFunc func(containerName string) map[string]string // ContainerLabelsFunc defines all base labels and their values attached to
// each metric exported by cAdvisor.
type ContainerLabelsFunc func(*info.ContainerInfo) map[string]string
// PrometheusCollector implements prometheus.Collector. // PrometheusCollector implements prometheus.Collector.
type PrometheusCollector struct { type PrometheusCollector struct {
infoProvider infoProvider infoProvider infoProvider
errors prometheus.Gauge errors prometheus.Gauge
containerMetrics []containerMetric containerMetrics []containerMetric
containerNameToLabels ContainerNameToLabelsFunc containerLabelsFunc ContainerLabelsFunc
} }
// NewPrometheusCollector returns a new PrometheusCollector. // NewPrometheusCollector returns a new PrometheusCollector. The passed
func NewPrometheusCollector(infoProvider infoProvider, f ContainerNameToLabelsFunc) *PrometheusCollector { // ContainerLabelsFunc specifies which base labels will be attached to all
// exported metrics. If left to nil, the DefaultContainerLabels function
// will be used instead.
func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc) *PrometheusCollector {
if f == nil {
f = DefaultContainerLabels
}
c := &PrometheusCollector{ c := &PrometheusCollector{
infoProvider: infoProvider, infoProvider: i,
containerNameToLabels: f, containerLabelsFunc: f,
errors: prometheus.NewGauge(prometheus.GaugeOpts{ errors: prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "container", Namespace: "container",
Name: "scrape_error", Name: "scrape_error",
@ -533,10 +542,38 @@ func (c *PrometheusCollector) Collect(ch chan<- prometheus.Metric) {
} }
const ( const (
containerLabelPrefix = "container_label_" // ContainerLabelPrefix is the prefix added to all container labels.
containerEnvPrefix = "container_env_" ContainerLabelPrefix = "container_label_"
// ContainerEnvPrefix is the prefix added to all env variable labels.
ContainerEnvPrefix = "container_env_"
// LabelID is the name of the id label.
LabelID = "id"
// LabelName is the name of the name label.
LabelName = "name"
// LabelImage is the name of the image label.
LabelImage = "image"
) )
// DefaultContainerLabels implements ContainerLabelsFunc. It exports the
// container name, first alias, image name as well as all its env and label
// values.
func DefaultContainerLabels(container *info.ContainerInfo) map[string]string {
set := map[string]string{LabelID: container.Name}
if len(container.Aliases) > 0 {
set[LabelName] = container.Aliases[0]
}
if image := container.Spec.Image; len(image) > 0 {
set[LabelImage] = image
}
for k, v := range container.Spec.Labels {
set[ContainerLabelPrefix+k] = v
}
for k, v := range container.Spec.Envs {
set[ContainerEnvPrefix+k] = v
}
return set
}
func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) { func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) {
containers, err := c.infoProvider.SubcontainersInfo("/", &info.ContainerInfoRequest{NumStats: 1}) containers, err := c.infoProvider.SubcontainersInfo("/", &info.ContainerInfoRequest{NumStats: 1})
if err != nil { if err != nil {
@ -545,56 +582,32 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric)
return return
} }
for _, container := range containers { for _, container := range containers {
baseLabels := []string{"id"} labels, values := []string{}, []string{}
id := container.Name for l, v := range c.containerLabelsFunc(container) {
name := id labels = append(labels, sanitizeLabelName(l))
if len(container.Aliases) > 0 { values = append(values, v)
name = container.Aliases[0]
baseLabels = append(baseLabels, "name")
}
image := container.Spec.Image
if len(image) > 0 {
baseLabels = append(baseLabels, "image")
}
baseLabelValues := []string{id, name, image}[:len(baseLabels)]
if c.containerNameToLabels != nil {
newLabels := c.containerNameToLabels(name)
for k, v := range newLabels {
baseLabels = append(baseLabels, sanitizeLabelName(k))
baseLabelValues = append(baseLabelValues, v)
}
}
for k, v := range container.Spec.Labels {
baseLabels = append(baseLabels, sanitizeLabelName(containerLabelPrefix+k))
baseLabelValues = append(baseLabelValues, v)
}
for k, v := range container.Spec.Envs {
baseLabels = append(baseLabels, sanitizeLabelName(containerEnvPrefix+k))
baseLabelValues = append(baseLabelValues, v)
} }
// Container spec // Container spec
desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", baseLabels, nil) desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", labels, nil)
ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), baseLabelValues...) ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), values...)
if container.Spec.HasCpu { if container.Spec.HasCpu {
desc = prometheus.NewDesc("container_spec_cpu_period", "CPU period of the container.", baseLabels, nil) desc = prometheus.NewDesc("container_spec_cpu_period", "CPU period of the container.", labels, nil)
ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Period), baseLabelValues...) ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Period), values...)
if container.Spec.Cpu.Quota != 0 { if container.Spec.Cpu.Quota != 0 {
desc = prometheus.NewDesc("container_spec_cpu_quota", "CPU quota of the container.", baseLabels, nil) desc = prometheus.NewDesc("container_spec_cpu_quota", "CPU quota of the container.", labels, nil)
ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Quota), baseLabelValues...) ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Quota), values...)
} }
desc := prometheus.NewDesc("container_spec_cpu_shares", "CPU share of the container.", baseLabels, nil) desc := prometheus.NewDesc("container_spec_cpu_shares", "CPU share of the container.", labels, nil)
ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Limit), baseLabelValues...) ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Limit), values...)
} }
if container.Spec.HasMemory { if container.Spec.HasMemory {
desc := prometheus.NewDesc("container_spec_memory_limit_bytes", "Memory limit for the container.", baseLabels, nil) desc := prometheus.NewDesc("container_spec_memory_limit_bytes", "Memory limit for the container.", labels, nil)
ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Limit), baseLabelValues...) ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Limit), values...)
desc = prometheus.NewDesc("container_spec_memory_swap_limit_bytes", "Memory swap limit for the container.", baseLabels, nil) desc = prometheus.NewDesc("container_spec_memory_swap_limit_bytes", "Memory swap limit for the container.", labels, nil)
ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.SwapLimit), baseLabelValues...) ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.SwapLimit), values...)
} }
// Now for the actual metrics // Now for the actual metrics
@ -603,9 +616,9 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric)
if cm.condition != nil && !cm.condition(container.Spec) { if cm.condition != nil && !cm.condition(container.Spec) {
continue continue
} }
desc := cm.desc(baseLabels) desc := cm.desc(labels)
for _, metricValue := range cm.getValues(stats) { for _, metricValue := range cm.getValues(stats) {
ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append(baseLabelValues, metricValue.labels...)...) ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append(values, metricValue.labels...)...)
} }
} }
} }

18
metrics/prometheus_test.go
View File

@ -180,10 +180,10 @@ var (
) )
func TestPrometheusCollector(t *testing.T) { func TestPrometheusCollector(t *testing.T) {
c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(name string) map[string]string { c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(container *info.ContainerInfo) map[string]string {
return map[string]string{ s := DefaultContainerLabels(container)
"zone.name": "hello", s["zone.name"] = "hello"
} return s
}) })
prometheus.MustRegister(c) prometheus.MustRegister(c)
defer prometheus.Unregister(c) defer prometheus.Unregister(c)
@ -212,7 +212,7 @@ func testPrometheusCollector(t *testing.T, c *PrometheusCollector, metricsFile s
continue continue
} }
if want != gotLines[i] { if want != gotLines[i] {
t.Fatalf("want %s, got %s", want, gotLines[i]) t.Fatalf("unexpected metric line\nwant: %s\nhave: %s", want, gotLines[i])
} }
} }
} }
@ -250,10 +250,10 @@ func TestPrometheusCollector_scrapeFailure(t *testing.T) {
shouldFail: true, shouldFail: true,
} }
c := NewPrometheusCollector(provider, func(name string) map[string]string { c := NewPrometheusCollector(provider, func(container *info.ContainerInfo) map[string]string {
return map[string]string{ s := DefaultContainerLabels(container)
"zone.name": "hello", s["zone.name"] = "hello"
} return s
}) })
prometheus.MustRegister(c) prometheus.MustRegister(c)
defer prometheus.Unregister(c) defer prometheus.Unregister(c)