diff --git a/http/handlers.go b/http/handlers.go index 48e088da..420077d7 100644 --- a/http/handlers.go +++ b/http/handlers.go @@ -89,8 +89,11 @@ func RegisterHandlers(mux httpmux.Mux, containerManager manager.Manager, httpAut return nil } -func RegisterPrometheusHandler(mux httpmux.Mux, containerManager manager.Manager, prometheusEndpoint string, containerNameToLabelsFunc metrics.ContainerNameToLabelsFunc) { - collector := metrics.NewPrometheusCollector(containerManager, containerNameToLabelsFunc) +// RegisterPrometheusHandler creates a new PrometheusCollector, registers it +// on the global registry and configures the provided HTTP mux to handle the +// given Prometheus endpoint. +func RegisterPrometheusHandler(mux httpmux.Mux, containerManager manager.Manager, prometheusEndpoint string, f metrics.ContainerLabelsFunc) { + collector := metrics.NewPrometheusCollector(containerManager, f) prometheus.MustRegister(collector) mux.Handle(prometheusEndpoint, prometheus.Handler()) } diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 0e7e8fba..7d738599 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -25,13 +25,14 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -// This will usually be manager.Manager, but can be swapped out for testing. +// infoProvider will usually be manager.Manager, but can be swapped out for testing. type infoProvider interface { - // Get information about all subcontainers of the specified container (includes self). + // SubcontainersInfo provides information about all subcontainers of the + // specified container including itself. SubcontainersInfo(containerName string, query *info.ContainerInfoRequest) ([]*info.ContainerInfo, error) - // Get information about the version. + // GetVersionInfo provides information about the version. GetVersionInfo() (*info.VersionInfo, error) - // Get information about the machine. + // GetMachineInfo provides information about the machine. GetMachineInfo() (*info.MachineInfo, error) } @@ -56,8 +57,8 @@ func fsValues(fsStats []info.FsStats, valueFn func(*info.FsStats) float64) metri return values } -// A containerMetric describes a multi-dimensional metric used for exposing -// a certain type of container statistic. +// containerMetric describes a multi-dimensional metric used for exposing a +// certain type of container statistic. type containerMetric struct { name string help string @@ -71,21 +72,29 @@ func (cm *containerMetric) desc(baseLabels []string) *prometheus.Desc { return prometheus.NewDesc(cm.name, cm.help, append(baseLabels, cm.extraLabels...), nil) } -type ContainerNameToLabelsFunc func(containerName string) map[string]string +// ContainerLabelsFunc defines all base labels and their values attached to +// each metric exported by cAdvisor. +type ContainerLabelsFunc func(*info.ContainerInfo) map[string]string // PrometheusCollector implements prometheus.Collector. type PrometheusCollector struct { - infoProvider infoProvider - errors prometheus.Gauge - containerMetrics []containerMetric - containerNameToLabels ContainerNameToLabelsFunc + infoProvider infoProvider + errors prometheus.Gauge + containerMetrics []containerMetric + containerLabelsFunc ContainerLabelsFunc } -// NewPrometheusCollector returns a new PrometheusCollector. -func NewPrometheusCollector(infoProvider infoProvider, f ContainerNameToLabelsFunc) *PrometheusCollector { +// NewPrometheusCollector returns a new PrometheusCollector. The passed +// ContainerLabelsFunc specifies which base labels will be attached to all +// exported metrics. If left to nil, the DefaultContainerLabels function +// will be used instead. +func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc) *PrometheusCollector { + if f == nil { + f = DefaultContainerLabels + } c := &PrometheusCollector{ - infoProvider: infoProvider, - containerNameToLabels: f, + infoProvider: i, + containerLabelsFunc: f, errors: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "container", Name: "scrape_error", @@ -533,10 +542,38 @@ func (c *PrometheusCollector) Collect(ch chan<- prometheus.Metric) { } const ( - containerLabelPrefix = "container_label_" - containerEnvPrefix = "container_env_" + // ContainerLabelPrefix is the prefix added to all container labels. + ContainerLabelPrefix = "container_label_" + // ContainerEnvPrefix is the prefix added to all env variable labels. + ContainerEnvPrefix = "container_env_" + // LabelID is the name of the id label. + LabelID = "id" + // LabelName is the name of the name label. + LabelName = "name" + // LabelImage is the name of the image label. + LabelImage = "image" ) +// DefaultContainerLabels implements ContainerLabelsFunc. It exports the +// container name, first alias, image name as well as all its env and label +// values. +func DefaultContainerLabels(container *info.ContainerInfo) map[string]string { + set := map[string]string{LabelID: container.Name} + if len(container.Aliases) > 0 { + set[LabelName] = container.Aliases[0] + } + if image := container.Spec.Image; len(image) > 0 { + set[LabelImage] = image + } + for k, v := range container.Spec.Labels { + set[ContainerLabelPrefix+k] = v + } + for k, v := range container.Spec.Envs { + set[ContainerEnvPrefix+k] = v + } + return set +} + func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) { containers, err := c.infoProvider.SubcontainersInfo("/", &info.ContainerInfoRequest{NumStats: 1}) if err != nil { @@ -545,56 +582,32 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) return } for _, container := range containers { - baseLabels := []string{"id"} - id := container.Name - name := id - if len(container.Aliases) > 0 { - name = container.Aliases[0] - baseLabels = append(baseLabels, "name") - } - image := container.Spec.Image - if len(image) > 0 { - baseLabels = append(baseLabels, "image") - } - baseLabelValues := []string{id, name, image}[:len(baseLabels)] - - if c.containerNameToLabels != nil { - newLabels := c.containerNameToLabels(name) - for k, v := range newLabels { - baseLabels = append(baseLabels, sanitizeLabelName(k)) - baseLabelValues = append(baseLabelValues, v) - } - } - - for k, v := range container.Spec.Labels { - baseLabels = append(baseLabels, sanitizeLabelName(containerLabelPrefix+k)) - baseLabelValues = append(baseLabelValues, v) - } - for k, v := range container.Spec.Envs { - baseLabels = append(baseLabels, sanitizeLabelName(containerEnvPrefix+k)) - baseLabelValues = append(baseLabelValues, v) + labels, values := []string{}, []string{} + for l, v := range c.containerLabelsFunc(container) { + labels = append(labels, sanitizeLabelName(l)) + values = append(values, v) } // Container spec - desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), baseLabelValues...) + desc := prometheus.NewDesc("container_start_time_seconds", "Start time of the container since unix epoch in seconds.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.CreationTime.Unix()), values...) if container.Spec.HasCpu { - desc = prometheus.NewDesc("container_spec_cpu_period", "CPU period of the container.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Period), baseLabelValues...) + desc = prometheus.NewDesc("container_spec_cpu_period", "CPU period of the container.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Period), values...) if container.Spec.Cpu.Quota != 0 { - desc = prometheus.NewDesc("container_spec_cpu_quota", "CPU quota of the container.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Quota), baseLabelValues...) + desc = prometheus.NewDesc("container_spec_cpu_quota", "CPU quota of the container.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Quota), values...) } - desc := prometheus.NewDesc("container_spec_cpu_shares", "CPU share of the container.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Limit), baseLabelValues...) + desc := prometheus.NewDesc("container_spec_cpu_shares", "CPU share of the container.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(container.Spec.Cpu.Limit), values...) } if container.Spec.HasMemory { - desc := prometheus.NewDesc("container_spec_memory_limit_bytes", "Memory limit for the container.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Limit), baseLabelValues...) - desc = prometheus.NewDesc("container_spec_memory_swap_limit_bytes", "Memory swap limit for the container.", baseLabels, nil) - ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.SwapLimit), baseLabelValues...) + desc := prometheus.NewDesc("container_spec_memory_limit_bytes", "Memory limit for the container.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.Limit), values...) + desc = prometheus.NewDesc("container_spec_memory_swap_limit_bytes", "Memory swap limit for the container.", labels, nil) + ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, specMemoryValue(container.Spec.Memory.SwapLimit), values...) } // Now for the actual metrics @@ -603,9 +616,9 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) if cm.condition != nil && !cm.condition(container.Spec) { continue } - desc := cm.desc(baseLabels) + desc := cm.desc(labels) for _, metricValue := range cm.getValues(stats) { - ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append(baseLabelValues, metricValue.labels...)...) + ch <- prometheus.MustNewConstMetric(desc, cm.valueType, float64(metricValue.value), append(values, metricValue.labels...)...) } } } diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go index 8c54097e..d62a64ec 100644 --- a/metrics/prometheus_test.go +++ b/metrics/prometheus_test.go @@ -180,10 +180,10 @@ var ( ) func TestPrometheusCollector(t *testing.T) { - c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(name string) map[string]string { - return map[string]string{ - "zone.name": "hello", - } + c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(container *info.ContainerInfo) map[string]string { + s := DefaultContainerLabels(container) + s["zone.name"] = "hello" + return s }) prometheus.MustRegister(c) defer prometheus.Unregister(c) @@ -212,7 +212,7 @@ func testPrometheusCollector(t *testing.T, c *PrometheusCollector, metricsFile s continue } if want != gotLines[i] { - t.Fatalf("want %s, got %s", want, gotLines[i]) + t.Fatalf("unexpected metric line\nwant: %s\nhave: %s", want, gotLines[i]) } } } @@ -250,10 +250,10 @@ func TestPrometheusCollector_scrapeFailure(t *testing.T) { shouldFail: true, } - c := NewPrometheusCollector(provider, func(name string) map[string]string { - return map[string]string{ - "zone.name": "hello", - } + c := NewPrometheusCollector(provider, func(container *info.ContainerInfo) map[string]string { + s := DefaultContainerLabels(container) + s["zone.name"] = "hello" + return s }) prometheus.MustRegister(c) defer prometheus.Unregister(c)