diff --git a/metrics/prometheus.go b/metrics/prometheus.go index a8e6abb2..a1908285 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -16,6 +16,7 @@ package metrics import ( "fmt" + "regexp" "time" info "github.com/google/cadvisor/info/v1" @@ -508,13 +509,13 @@ func (c *PrometheusCollector) collectContainersInfo(ch chan<- prometheus.Metric) if c.containerNameToLabels != nil { newLabels := c.containerNameToLabels(name) for k, v := range newLabels { - baseLabels = append(baseLabels, k) + baseLabels = append(baseLabels, sanitizeLabelName(k)) baseLabelValues = append(baseLabelValues, v) } } for labelKey, labelValue := range container.Spec.Labels { - baseLabels = append(baseLabels, labelKey) + baseLabels = append(baseLabels, sanitizeLabelName(labelKey)) baseLabelValues = append(baseLabelValues, labelValue) } @@ -576,3 +577,11 @@ func specMemoryValue(v uint64) float64 { } return float64(v) } + +var invalidLabelCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`) + +// sanitizeLabelName replaces anything that doesn't match +// client_label.LabelNameRE with an underscore. +func sanitizeLabelName(name string) string { + return invalidLabelCharRE.ReplaceAllString(name, "_") +} diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go index cf45f510..e22b8d5d 100644 --- a/metrics/prometheus_test.go +++ b/metrics/prometheus_test.go @@ -58,7 +58,7 @@ func (p testSubcontainersInfoProvider) SubcontainersInfo(string, *info.Container Image: "test", CreationTime: time.Unix(1257894000, 0), Labels: map[string]string{ - "foo": "bar", + "foo.metric": "bar", }, }, Stats: []*info.ContainerStats{ @@ -162,7 +162,13 @@ var ( ) func TestPrometheusCollector(t *testing.T) { - prometheus.MustRegister(NewPrometheusCollector(testSubcontainersInfoProvider{}, nil)) + c := NewPrometheusCollector(testSubcontainersInfoProvider{}, func(name string) map[string]string { + return map[string]string{ + "zone.name": "hello", + } + }) + prometheus.MustRegister(c) + defer prometheus.Unregister(c) rw := httptest.NewRecorder() prometheus.Handler().ServeHTTP(rw, &http.Request{}) diff --git a/metrics/testdata/prometheus_metrics b/metrics/testdata/prometheus_metrics index 54877ec2..32020257 100644 --- a/metrics/testdata/prometheus_metrics +++ b/metrics/testdata/prometheus_metrics @@ -3,123 +3,123 @@ cadvisor_version_info{cadvisorRevision="abcdef",cadvisorVersion="0.16.0",dockerVersion="1.8.1",kernelVersion="4.1.6-200.fc22.x86_64",osVersion="Fedora 22 (Twenty Two)"} 1 # HELP container_cpu_system_seconds_total Cumulative system cpu time consumed in seconds. # TYPE container_cpu_system_seconds_total counter -container_cpu_system_seconds_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 7e-09 +container_cpu_system_seconds_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 7e-09 # HELP container_cpu_usage_seconds_total Cumulative cpu time consumed per cpu in seconds. # TYPE container_cpu_usage_seconds_total counter -container_cpu_usage_seconds_total{cpu="cpu00",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 2e-09 -container_cpu_usage_seconds_total{cpu="cpu01",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 3e-09 -container_cpu_usage_seconds_total{cpu="cpu02",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4e-09 -container_cpu_usage_seconds_total{cpu="cpu03",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 5e-09 +container_cpu_usage_seconds_total{cpu="cpu00",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2e-09 +container_cpu_usage_seconds_total{cpu="cpu01",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 3e-09 +container_cpu_usage_seconds_total{cpu="cpu02",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4e-09 +container_cpu_usage_seconds_total{cpu="cpu03",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 5e-09 # HELP container_cpu_user_seconds_total Cumulative user cpu time consumed in seconds. # TYPE container_cpu_user_seconds_total counter -container_cpu_user_seconds_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 6e-09 +container_cpu_user_seconds_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 6e-09 # HELP container_fs_io_current Number of I/Os currently in progress # TYPE container_fs_io_current gauge -container_fs_io_current{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 42 -container_fs_io_current{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 47 +container_fs_io_current{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 42 +container_fs_io_current{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 47 # HELP container_fs_io_time_seconds_total Cumulative count of seconds spent doing I/Os # TYPE container_fs_io_time_seconds_total counter -container_fs_io_time_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.3e-08 -container_fs_io_time_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.8e-08 +container_fs_io_time_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.3e-08 +container_fs_io_time_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.8e-08 # HELP container_fs_io_time_weighted_seconds_total Cumulative weighted I/O time in seconds # TYPE container_fs_io_time_weighted_seconds_total counter -container_fs_io_time_weighted_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.4e-08 -container_fs_io_time_weighted_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.9e-08 +container_fs_io_time_weighted_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.4e-08 +container_fs_io_time_weighted_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.9e-08 # HELP container_fs_limit_bytes Number of bytes that can be consumed by the container on this filesystem. # TYPE container_fs_limit_bytes gauge -container_fs_limit_bytes{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 22 -container_fs_limit_bytes{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 37 +container_fs_limit_bytes{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 22 +container_fs_limit_bytes{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 37 # HELP container_fs_read_seconds_total Cumulative count of seconds spent reading # TYPE container_fs_read_seconds_total counter -container_fs_read_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 2.7e-08 -container_fs_read_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.2e-08 +container_fs_read_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 2.7e-08 +container_fs_read_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.2e-08 # HELP container_fs_reads_merged_total Cumulative count of reads merged # TYPE container_fs_reads_merged_total counter -container_fs_reads_merged_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 25 -container_fs_reads_merged_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 40 +container_fs_reads_merged_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 25 +container_fs_reads_merged_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 # HELP container_fs_reads_total Cumulative count of reads completed # TYPE container_fs_reads_total counter -container_fs_reads_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 24 -container_fs_reads_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 39 +container_fs_reads_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 24 +container_fs_reads_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 # HELP container_fs_sector_reads_total Cumulative count of sector reads completed # TYPE container_fs_sector_reads_total counter -container_fs_sector_reads_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 26 -container_fs_sector_reads_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 41 +container_fs_sector_reads_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 26 +container_fs_sector_reads_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 41 # HELP container_fs_sector_writes_total Cumulative count of sector writes completed # TYPE container_fs_sector_writes_total counter -container_fs_sector_writes_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 40 -container_fs_sector_writes_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 45 +container_fs_sector_writes_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 40 +container_fs_sector_writes_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 45 # HELP container_fs_usage_bytes Number of bytes that are consumed by the container on this filesystem. # TYPE container_fs_usage_bytes gauge -container_fs_usage_bytes{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 23 -container_fs_usage_bytes{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 38 +container_fs_usage_bytes{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 23 +container_fs_usage_bytes{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 38 # HELP container_fs_write_seconds_total Cumulative count of seconds spent writing # TYPE container_fs_write_seconds_total counter -container_fs_write_seconds_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.1e-08 -container_fs_write_seconds_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 4.6e-08 +container_fs_write_seconds_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.1e-08 +container_fs_write_seconds_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 4.6e-08 # HELP container_fs_writes_merged_total Cumulative count of writes merged # TYPE container_fs_writes_merged_total counter -container_fs_writes_merged_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 39 -container_fs_writes_merged_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 44 +container_fs_writes_merged_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 39 +container_fs_writes_merged_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 44 # HELP container_fs_writes_total Cumulative count of writes completed # TYPE container_fs_writes_total counter -container_fs_writes_total{device="sda1",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 28 -container_fs_writes_total{device="sda2",foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 43 +container_fs_writes_total{device="sda1",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 28 +container_fs_writes_total{device="sda2",foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 43 # HELP container_last_seen Last time a container was seen by the exporter # TYPE container_last_seen gauge -container_last_seen{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 1.426203694e+09 +container_last_seen{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.426203694e+09 # HELP container_memory_failcnt Number of memory usage hits limits # TYPE container_memory_failcnt counter -container_memory_failcnt{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 0 +container_memory_failcnt{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 0 # HELP container_memory_failures_total Cumulative count of memory allocation failures. # TYPE container_memory_failures_total counter -container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault"} 10 -container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault"} 11 -container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault"} 12 -container_memory_failures_total{foo="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault"} 13 +container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgfault",zone_name="hello"} 10 +container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="container",type="pgmajfault",zone_name="hello"} 11 +container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgfault",zone_name="hello"} 12 +container_memory_failures_total{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",scope="hierarchy",type="pgmajfault",zone_name="hello"} 13 # HELP container_memory_usage_bytes Current memory usage in bytes. # TYPE container_memory_usage_bytes gauge -container_memory_usage_bytes{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 8 +container_memory_usage_bytes{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 8 # HELP container_memory_working_set_bytes Current working set in bytes. # TYPE container_memory_working_set_bytes gauge -container_memory_working_set_bytes{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 9 +container_memory_working_set_bytes{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 9 # HELP container_network_receive_bytes_total Cumulative count of bytes received # TYPE container_network_receive_bytes_total counter -container_network_receive_bytes_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 14 +container_network_receive_bytes_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 14 # HELP container_network_receive_errors_total Cumulative count of errors encountered while receiving # TYPE container_network_receive_errors_total counter -container_network_receive_errors_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 16 +container_network_receive_errors_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 16 # HELP container_network_receive_packets_dropped_total Cumulative count of packets dropped while receiving # TYPE container_network_receive_packets_dropped_total counter -container_network_receive_packets_dropped_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 17 +container_network_receive_packets_dropped_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 17 # HELP container_network_receive_packets_total Cumulative count of packets received # TYPE container_network_receive_packets_total counter -container_network_receive_packets_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 15 +container_network_receive_packets_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 15 # HELP container_network_transmit_bytes_total Cumulative count of bytes transmitted # TYPE container_network_transmit_bytes_total counter -container_network_transmit_bytes_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 18 +container_network_transmit_bytes_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 18 # HELP container_network_transmit_errors_total Cumulative count of errors encountered while transmitting # TYPE container_network_transmit_errors_total counter -container_network_transmit_errors_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 20 +container_network_transmit_errors_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 20 # HELP container_network_transmit_packets_dropped_total Cumulative count of packets dropped while transmitting # TYPE container_network_transmit_packets_dropped_total counter -container_network_transmit_packets_dropped_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 21 +container_network_transmit_packets_dropped_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 21 # HELP container_network_transmit_packets_total Cumulative count of packets transmitted # TYPE container_network_transmit_packets_total counter -container_network_transmit_packets_total{foo="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias"} 19 +container_network_transmit_packets_total{foo_metric="bar",id="testcontainer",image="test",interface="eth0",name="testcontaineralias",zone_name="hello"} 19 # HELP container_scrape_error 1 if there was an error while getting container metrics, 0 otherwise # TYPE container_scrape_error gauge container_scrape_error 0 # HELP container_start_time_seconds Start time of the container since unix epoch in seconds. # TYPE container_start_time_seconds gauge -container_start_time_seconds{foo="bar",id="testcontainer",image="test",name="testcontaineralias"} 1.257894e+09 +container_start_time_seconds{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",zone_name="hello"} 1.257894e+09 # HELP container_tasks_state Number of tasks in given state # TYPE container_tasks_state gauge -container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="iowaiting"} 54 -container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="running"} 51 -container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="sleeping"} 50 -container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="stopped"} 52 -container_tasks_state{foo="bar",id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible"} 53 +container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="iowaiting",zone_name="hello"} 54 +container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="running",zone_name="hello"} 51 +container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="sleeping",zone_name="hello"} 50 +container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="stopped",zone_name="hello"} 52 +container_tasks_state{foo_metric="bar",id="testcontainer",image="test",name="testcontaineralias",state="uninterruptible",zone_name="hello"} 53 # HELP http_request_duration_microseconds The HTTP request latencies in microseconds. # TYPE http_request_duration_microseconds summary http_request_duration_microseconds{handler="prometheus",quantile="0.5"} 0