From b804ceac300cf41309909f2eca43c66549c013a5 Mon Sep 17 00:00:00 2001 From: Nan Deng Date: Tue, 29 Jul 2014 14:27:52 -0700 Subject: [PATCH] read core loads from /proc/sched_debug --- utils/procfs/scheddebug.go | 204 ++++++++++ utils/procfs/scheddebug_test.go | 657 ++++++++++++++++++++++++++++++++ 2 files changed, 861 insertions(+) create mode 100644 utils/procfs/scheddebug.go create mode 100644 utils/procfs/scheddebug_test.go diff --git a/utils/procfs/scheddebug.go b/utils/procfs/scheddebug.go new file mode 100644 index 00000000..0aa67f96 --- /dev/null +++ b/utils/procfs/scheddebug.go @@ -0,0 +1,204 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import ( + "bufio" + "fmt" + "strings" + + "github.com/google/cadvisor/utils/fs" +) + +type SchedulerLoadReader interface { + // Load() returns the load of each core of given container. If there's + // no information of given container, it should return nil, nil. If the + // returned load is not nil, the number of elements in the returned + // slice must be the same as the number of cores on the machine. Each + // element in the returned slices represents number of tasks/threads in + // the container and waiting for the CPU, i.e. number of runnable + // tasks. + Load(container string) ([]int, error) + + AllContainers() ([]string, error) +} + +func NewSchedulerLoadReader() (SchedulerLoadReader, error) { + schedDebug, err := fs.Open("/proc/sched_debug") + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(schedDebug) + stateMachine := newSchedDebugReader() + for scanner.Scan() { + line := scanner.Text() + err = stateMachine.ProcessLine(line) + if err != nil { + return nil, err + } + } + if err = scanner.Err(); err != nil { + return nil, err + } + return stateMachine.Load() +} + +type schedDebugReaderStateMachine struct { + currentState schedDebugReaderState + context *schedDebugContext +} + +func newSchedDebugReader() *schedDebugReaderStateMachine { + return &schedDebugReaderStateMachine{ + currentState: &schedDebugReaderStateReadingVersion{}, + context: &schedDebugContext{ + loadMap: make(map[string][]int, 8), + }, + } +} + +func (self *schedDebugReaderStateMachine) ProcessLine(line string) error { + nextState, err := self.currentState.Transit(self.context, line) + if err != nil { + return err + } + self.currentState = nextState + return nil +} + +func (self *schedDebugReaderStateMachine) Load() (SchedulerLoadReader, error) { + return self.context.loadMap, nil +} + +type schedDebugContext struct { + loadMap simpleSchedulerLoadReader + numCores int +} + +// Key: container name +// Value: number of runnable processes in each core +type simpleSchedulerLoadReader map[string][]int + +func (self simpleSchedulerLoadReader) Load(container string) ([]int, error) { + if load, ok := self[container]; ok { + return load, nil + } + return nil, nil +} + +func (self simpleSchedulerLoadReader) AllContainers() ([]string, error) { + ret := make([]string, 0, len(self)) + for c, _ := range self { + ret = append(ret, c) + } + return ret, nil +} + +// This interface is used to represent a state when reading sched_debug. The +// reader of sched_debug is a state machine and will transit to another state +// (and change its behavior accordingly) when a certain line is read. +// The state machine is implemented using the State Pattern. +type schedDebugReaderState interface { + // Transit() consumes a line from sched_debug and returns a new state + // for the next line. It may change the context accordingly. + Transit(context *schedDebugContext, line string) (schedDebugReaderState, error) +} + +// State: Reading version +// In this state, the state machine is waiting for the version of the Sched Debug message. +// If the version is supported, it will transit to WaitingHeader state. +// Otherwise, it will report error +type schedDebugReaderStateReadingVersion struct { +} + +func (self *schedDebugReaderStateReadingVersion) Transit(context *schedDebugContext, line string) (schedDebugReaderState, error) { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "Sched Debug Version") { + if !strings.HasPrefix(line, "Sched Debug Version: v0.11") { + return nil, fmt.Errorf("unsupported sched_debug version: %v", line) + } + return &schedDebugReaderStateWaitingHeader{}, nil + } + return self, nil +} + +// State: WaitingHeader +// In this state the state machine is waiting for the header of the stats table. +// It will transit to ReadingTask state once it received a header. +type schedDebugReaderStateWaitingHeader struct { +} + +func (self *schedDebugReaderStateWaitingHeader) isSeparator(line string) bool { + line = strings.TrimSpace(line) + if len(line) == 0 { + return false + } + for _, ch := range line { + if ch != '-' { + return false + } + } + return true +} + +func (self *schedDebugReaderStateWaitingHeader) Transit(context *schedDebugContext, line string) (schedDebugReaderState, error) { + if self.isSeparator(line) { + // A new table of runnable tasks + context.numCores++ + loadMap := make(map[string][]int, len(context.loadMap)) + for container, loads := range context.loadMap { + for len(loads) < context.numCores { + loads = append(loads, 0) + } + loadMap[container] = loads + } + context.loadMap = loadMap + ret := &schedDebugReaderStateReadingTasks{} + return ret, nil + } + return self, nil +} + +// State: ReadingTasks +// In this state, the state machine expects that each line contains a stats info of a thread. +// It will transit to WaitingHeader state once it reads an empty line. +type schedDebugReaderStateReadingTasks struct { +} + +func (self *schedDebugReaderStateReadingTasks) Transit(context *schedDebugContext, line string) (schedDebugReaderState, error) { + line = strings.TrimSpace(line) + if len(line) == 0 { + // End of a table. Waiting for another table. + ret := &schedDebugReaderStateWaitingHeader{} + return ret, nil + } + fields := strings.Fields(line) + + // Example of one row: + // rcuos/2 10 153908711.511953 9697302 120 153908711.511953 462044.084299 4409304806.819551 0 / + + container := fields[len(fields)-1] + var loads []int + ok := false + if loads, ok = context.loadMap[container]; !ok { + loads = make([]int, context.numCores) + } + for len(loads) < context.numCores { + loads = append(loads, 0) + } + loads[len(loads)-1]++ + context.loadMap[container] = loads + return self, nil +} diff --git a/utils/procfs/scheddebug_test.go b/utils/procfs/scheddebug_test.go new file mode 100644 index 00000000..b91c4d2c --- /dev/null +++ b/utils/procfs/scheddebug_test.go @@ -0,0 +1,657 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import ( + "reflect" + "testing" + + "code.google.com/p/gomock/gomock" + + "github.com/google/cadvisor/utils/fs" + "github.com/google/cadvisor/utils/fs/mockfs" +) + +var schedDebugToLoadsPerContainerPerCore = []struct { + SchedDebugContent string + Loads map[string][]int + Error error +}{ + { + ` +Sched Debug Version: v0.11, 3.13.0-29-generic #53-Ubuntu +runnable tasks: + task PID tree-key switches prio exec-runtime sum-exec sum-sleep +---------------------------------------------------------------------------------------------------------- + kthreadd 2 159918906.381680 397 120 159918906.381680 16.755308 4408983057.115372 0 / + kworker/0:0 9225 159645585.744016 8 120 159645585.744016 0.151033 9178573.870160 0 / + cadvisor 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/hash + someproc 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/somecontainer + +runnable tasks: + task PID tree-key switches prio exec-runtime sum-exec sum-sleep +---------------------------------------------------------------------------------------------------------- + kworker/0:0 9225 159645585.744016 8 120 159645585.744016 0.151033 9178573.870160 0 / + cadvisor 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/hash + other 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/another + + `, + map[string][]int{ + "/": {2, 1}, + "/docker/hash": {1, 1}, + "/docker/another": {0, 1}, + "/docker/somecontainer": {1, 0}, + }, + nil, + }, + { + ` +Sched Debug Version: v0.11, 3.13.0-29-generic #53-Ubuntu +ktime : 4409767725.360719 +sched_clk : 4722208509.060012 +cpu_clk : 4409767725.360725 +jiffies : 5397334228 +sched_clock_stable : 0 + +sysctl_sched + .sysctl_sched_latency : 24.000000 + .sysctl_sched_min_granularity : 3.000000 + .sysctl_sched_wakeup_granularity : 4.000000 + .sysctl_sched_child_runs_first : 0 + .sysctl_sched_features : 77435 + .sysctl_sched_tunable_scaling : 1 (logaritmic) + +cpu#0, 2599.998 MHz + .nr_running : 0 + .load : 0 + .nr_switches : 4015586556 + .nr_load_updates : 105987951 + .nr_uninterruptible : -334192 + .next_balance : 5397.334205 + .curr->pid : 0 + .clock : 4409767641.399025 + .cpu_load[0] : 0 + .cpu_load[1] : 0 + .cpu_load[2] : 0 + .cpu_load[3] : 0 + .cpu_load[4] : 0 + .yld_count : 1327594 + .sched_count : -278003554 + .sched_goidle : 2002340573 + .avg_idle : 187229 + .ttwu_count : 2078272826 + .ttwu_local : 1180909486 + +cfs_rq[0]:/user/1014.user + .exec_clock : 13335742.850392 + .MIN_vruntime : 0.000001 + .min_vruntime : 19737907.734191 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -140209124.399553 + .nr_spread_over : 0 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 3 + .tg_load_contrib : 1 + .tg_runnable_contrib : 5 + .tg_load_avg : 12 + .tg->runnable_avg : 40 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .se->exec_start : 4409767641.399025 + .se->vruntime : 22521033.619085 + .se->sum_exec_runtime : 13335742.878469 + .se->statistics.wait_start : 0.000000 + .se->statistics.sleep_start : 0.000000 + .se->statistics.block_start : 0.000000 + .se->statistics.sleep_max : 0.000000 + .se->statistics.block_max : 0.000000 + .se->statistics.exec_max : 1176.006499 + .se->statistics.slice_max : 382.761845 + .se->statistics.wait_max : 202.195062 + .se->statistics.wait_sum : 16916.088365 + .se->statistics.wait_count : 286435218 + .se->load.weight : 2 + .se->avg.runnable_avg_sum : 306 + .se->avg.runnable_avg_period : 46456 + .se->avg.load_avg_contrib : 1 + .se->avg.decay_count : 4205482141 + +cfs_rq[0]:/user + .exec_clock : 14007754.294731 + .MIN_vruntime : 0.000001 + .min_vruntime : 22521033.619085 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -137425998.514659 + .nr_spread_over : 0 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 1 + .tg_load_contrib : 5 + .tg_runnable_contrib : 4 + .tg_load_avg : 15 + .tg->runnable_avg : 43 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .se->exec_start : 4409767641.399025 + .se->vruntime : 159947032.133744 + .se->sum_exec_runtime : 14007766.619125 + .se->statistics.wait_start : 0.000000 + .se->statistics.sleep_start : 0.000000 + .se->statistics.block_start : 0.000000 + .se->statistics.sleep_max : 0.000000 + .se->statistics.block_max : 0.000000 + .se->statistics.exec_max : 1176.006499 + .se->statistics.slice_max : 8.170829 + .se->statistics.wait_max : 202.195062 + .se->statistics.wait_sum : 19616.722323 + .se->statistics.wait_count : 291817014 + .se->load.weight : 2 + .se->avg.runnable_avg_sum : 308 + .se->avg.runnable_avg_period : 46789 + .se->avg.load_avg_contrib : 7 + .se->avg.decay_count : 4205482141 + +cfs_rq[0]:/ + .exec_clock : 88845316.977008 + .MIN_vruntime : 0.000001 + .min_vruntime : 159947032.133744 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : 0.000000 + .nr_spread_over : 99 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 7 + .tg_load_contrib : 15 + .tg_runnable_contrib : 6 + .tg_load_avg : 42 + .tg->runnable_avg : 63 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .avg->runnable_avg_sum : 300 + .avg->runnable_avg_period : 46798 + +cfs_rq[0]:/user/1014.user/127.session + .exec_clock : 230108.794316 + .MIN_vruntime : 0.000001 + .min_vruntime : 220535.926524 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -159726496.207220 + .nr_spread_over : 38 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 9 + .tg_load_contrib : 7 + .tg_runnable_contrib : 5 + .tg_load_avg : 33 + .tg->runnable_avg : 23 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .se->exec_start : 4409767641.399025 + .se->vruntime : 19737907.734191 + .se->sum_exec_runtime : 230108.797063 + .se->statistics.wait_start : 0.000000 + .se->statistics.sleep_start : 0.000000 + .se->statistics.block_start : 0.000000 + .se->statistics.sleep_max : 0.000000 + .se->statistics.block_max : 0.000000 + .se->statistics.exec_max : 6.318158 + .se->statistics.slice_max : 6.060779 + .se->statistics.wait_max : 3.122343 + .se->statistics.wait_sum : 1247.416895 + .se->statistics.wait_count : 2198718 + .se->load.weight : 2 + .se->avg.runnable_avg_sum : 305 + .se->avg.runnable_avg_period : 46769 + .se->avg.load_avg_contrib : 3 + .se->avg.decay_count : 4205482141 + +rt_rq[0]: + .rt_nr_running : 0 + .rt_throttled : 0 + .rt_time : 0.000000 + .rt_runtime : 950.000000 + + +runnable tasks: + task PID tree-key switches prio exec-runtime sum-exec sum-sleep +---------------------------------------------------------------------------------------------------------- + kthreadd 2 159918906.381680 397 120 159918906.381680 16.755308 4408983057.115372 0 / + kworker/0:0 9225 159645585.744016 8 120 159645585.744016 0.151033 9178573.870160 0 / + cadvisor 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/hash + someproc 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/somecontainer + +cpu#1, 2599.998 MHz + .nr_running : 0 + .load : 0 + .nr_switches : 3944986693 + .nr_load_updates : 96182719 + .nr_uninterruptible : 173423 + .next_balance : 5397.334255 + .curr->pid : 0 + .clock : 4409767724.343465 + .cpu_load[0] : 0 + .cpu_load[1] : 0 + .cpu_load[2] : 0 + .cpu_load[3] : 0 + .cpu_load[4] : 0 + .yld_count : 1321983 + .sched_count : -348608544 + .sched_goidle : 1968452376 + .avg_idle : 1000000 + .ttwu_count : 1957923792 + .ttwu_local : 1102567359 + +cfs_rq[1]:/user/1014.user + .exec_clock : 13485925.121019 + .MIN_vruntime : 0.000001 + .min_vruntime : 19845478.053952 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -140101554.079792 + .nr_spread_over : 0 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 0 + .tg_load_contrib : 0 + .tg_runnable_contrib : 12 + .tg_load_avg : 12 + .tg->runnable_avg : 40 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .se->exec_start : 4409767724.340638 + .se->vruntime : 22532500.804117 + .se->sum_exec_runtime : 13485925.213913 + .se->statistics.wait_start : 0.000000 + .se->statistics.sleep_start : 0.000000 + .se->statistics.block_start : 0.000000 + .se->statistics.sleep_max : 0.000000 + .se->statistics.block_max : 0.000000 + .se->statistics.exec_max : 13.643888 + .se->statistics.slice_max : 489.490466 + .se->statistics.wait_max : 550.414020 + .se->statistics.wait_sum : 17330.937081 + .se->statistics.wait_count : 295976777 + .se->load.weight : 2 + .se->avg.runnable_avg_sum : 560 + .se->avg.runnable_avg_period : 47228 + .se->avg.load_avg_contrib : 0 + .se->avg.decay_count : 4205482221 + +cfs_rq[1]:/user/1018.user + .exec_clock : 480956.077278 + .MIN_vruntime : 0.000001 + .min_vruntime : 607293.984023 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -159339738.149721 + .nr_spread_over : 0 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 0 + .tg_load_contrib : 0 + .tg_runnable_contrib : 0 + .tg_load_avg : 0 + .tg->runnable_avg : 2 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .se->exec_start : 4409767638.164369 + .se->vruntime : 22532512.239259 + .se->sum_exec_runtime : 480956.091761 + .se->statistics.wait_start : 0.000000 + .se->statistics.sleep_start : 0.000000 + .se->statistics.block_start : 0.000000 + .se->statistics.sleep_max : 0.000000 + .se->statistics.block_max : 0.000000 + .se->statistics.exec_max : 20.787279 + .se->statistics.slice_max : 128.534664 + .se->statistics.wait_max : 16.863694 + .se->statistics.wait_sum : 2144.675431 + .se->statistics.wait_count : 3454450 + .se->load.weight : 2 + .se->avg.runnable_avg_sum : 13 + .se->avg.runnable_avg_period : 46463 + .se->avg.load_avg_contrib : 0 + .se->avg.decay_count : 4205482138 + +cfs_rq[1]:/docker + .exec_clock : 33323750.675251 + .MIN_vruntime : 0.000001 + .min_vruntime : 56928736.465342 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -103018295.668402 + .nr_spread_over : 0 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 0 + .tg_load_contrib : 0 + .tg_runnable_contrib : 2 + .tg_load_avg : 0 + .tg->runnable_avg : 2 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .se->exec_start : 4409767693.642881 + .se->vruntime : 152694416.015894 + .se->sum_exec_runtime : 33323774.559318 + .se->statistics.wait_start : 0.000000 + .se->statistics.sleep_start : 0.000000 + .se->statistics.block_start : 0.000000 + .se->statistics.sleep_max : 0.000000 + .se->statistics.block_max : 0.000000 + .se->statistics.exec_max : 244.042161 + .se->statistics.slice_max : 751.402989 + .se->statistics.wait_max : 9.112128 + .se->statistics.wait_sum : 48336.965928 + .se->statistics.wait_count : 633354214 + .se->load.weight : 2 + .se->avg.runnable_avg_sum : 110 + .se->avg.runnable_avg_period : 46476 + .se->avg.load_avg_contrib : 0 + .se->avg.decay_count : 4205482191 + +cfs_rq[1]:/user + .exec_clock : 14090901.680875 + .MIN_vruntime : 0.000001 + .min_vruntime : 22532512.239259 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -137414519.894485 + .nr_spread_over : 0 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 0 + .tg_load_contrib : 0 + .tg_runnable_contrib : 1 + .tg_load_avg : 15 + .tg->runnable_avg : 43 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .se->exec_start : 4409767724.340638 + .se->vruntime : 152694428.387252 + .se->sum_exec_runtime : 14090908.503310 + .se->statistics.wait_start : 0.000000 + .se->statistics.sleep_start : 0.000000 + .se->statistics.block_start : 0.000000 + .se->statistics.sleep_max : 0.000000 + .se->statistics.block_max : 0.000000 + .se->statistics.exec_max : 20.787279 + .se->statistics.slice_max : 9.088991 + .se->statistics.wait_max : 550.414020 + .se->statistics.wait_sum : 19912.196787 + .se->statistics.wait_count : 301194069 + .se->load.weight : 2 + .se->avg.runnable_avg_sum : 608 + .se->avg.runnable_avg_period : 48047 + .se->avg.load_avg_contrib : 0 + .se->avg.decay_count : 4205482221 + +cfs_rq[1]:/ + .exec_clock : 85028436.453750 + .MIN_vruntime : 0.000001 + .min_vruntime : 152694428.387252 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -7252603.746492 + .nr_spread_over : 62 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 0 + .tg_load_contrib : 0 + .tg_runnable_contrib : 14 + .tg_load_avg : 42 + .tg->runnable_avg : 63 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .avg->runnable_avg_sum : 670 + .avg->runnable_avg_period : 46562 + +cfs_rq[1]:/docker/d1cebfddcf94a8018cbd73175b1484ab239943b5ee517529509bb7502181fd15 + .exec_clock : 209.852282 + .MIN_vruntime : 0.000001 + .min_vruntime : 208.680611 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -159946823.453133 + .nr_spread_over : 0 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 0 + .tg_load_contrib : 0 + .tg_runnable_contrib : 2 + .tg_load_avg : 0 + .tg->runnable_avg : 2 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .se->exec_start : 4409767693.642881 + .se->vruntime : 56928736.465342 + .se->sum_exec_runtime : 209.852282 + .se->statistics.wait_start : 0.000000 + .se->statistics.sleep_start : 0.000000 + .se->statistics.block_start : 0.000000 + .se->statistics.sleep_max : 0.000000 + .se->statistics.block_max : 0.000000 + .se->statistics.exec_max : 2.409911 + .se->statistics.slice_max : 2.123931 + .se->statistics.wait_max : 0.062006 + .se->statistics.wait_sum : 0.319593 + .se->statistics.wait_count : 1810 + .se->load.weight : 2 + .se->avg.runnable_avg_sum : 113 + .se->avg.runnable_avg_period : 47401 + .se->avg.load_avg_contrib : 0 + .se->avg.decay_count : 4205482191 + +cfs_rq[1]:/user/1018.user/102.session + .exec_clock : 295188.205699 + .MIN_vruntime : 0.000001 + .min_vruntime : 267319.397716 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -159679712.736028 + .nr_spread_over : 11 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 0 + .tg_load_contrib : 0 + .tg_runnable_contrib : 0 + .tg_load_avg : 10 + .tg->runnable_avg : 2 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .se->exec_start : 4409767638.164369 + .se->vruntime : 607293.984023 + .se->sum_exec_runtime : 295188.217342 + .se->statistics.wait_start : 0.000000 + .se->statistics.sleep_start : 0.000000 + .se->statistics.block_start : 0.000000 + .se->statistics.sleep_max : 0.000000 + .se->statistics.block_max : 0.000000 + .se->statistics.exec_max : 20.787279 + .se->statistics.slice_max : 9.711102 + .se->statistics.wait_max : 8.558728 + .se->statistics.wait_sum : 1146.954809 + .se->statistics.wait_count : 2361162 + .se->load.weight : 2 + .se->avg.runnable_avg_sum : 13 + .se->avg.runnable_avg_period : 47447 + .se->avg.load_avg_contrib : 0 + .se->avg.decay_count : 4205482138 + +cfs_rq[1]:/user/1014.user/128.session + .exec_clock : 20.815341 + .MIN_vruntime : 0.000001 + .min_vruntime : 271.766523 + .max_vruntime : 0.000001 + .spread : 0.000000 + .spread0 : -159946760.367221 + .nr_spread_over : 13 + .nr_running : 0 + .load : 0 + .runnable_load_avg : 0 + .blocked_load_avg : 1 + .tg_load_contrib : 1 + .tg_runnable_contrib : 0 + .tg_load_avg : 981 + .tg->runnable_avg : 8 + .tg->cfs_bandwidth.timer_active: 0 + .throttled : 0 + .throttle_count : 0 + .se->exec_start : 4409767724.340638 + .se->vruntime : 19845466.859551 + .se->sum_exec_runtime : 20.815341 + .se->statistics.wait_start : 0.000000 + .se->statistics.sleep_start : 0.000000 + .se->statistics.block_start : 0.000000 + .se->statistics.sleep_max : 0.000000 + .se->statistics.block_max : 0.000000 + .se->statistics.exec_max : 1.712933 + .se->statistics.slice_max : 0.542023 + .se->statistics.wait_max : 0.072066 + .se->statistics.wait_sum : 0.437682 + .se->statistics.wait_count : 104 + .se->load.weight : 2 + .se->avg.runnable_avg_sum : 570 + .se->avg.runnable_avg_period : 47945 + .se->avg.load_avg_contrib : 0 + .se->avg.decay_count : 4205482221 + +rt_rq[1]: + .rt_nr_running : 0 + .rt_throttled : 0 + .rt_time : 0.000000 + .rt_runtime : 950.000000 + +runnable tasks: + task PID tree-key switches prio exec-runtime sum-exec sum-sleep +---------------------------------------------------------------------------------------------------------- + kworker/0:0 9225 159645585.744016 8 120 159645585.744016 0.151033 9178573.870160 0 / + cadvisor 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/hash + other 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/another + + `, + map[string][]int{ + "/": {2, 1}, + "/docker/hash": {1, 1}, + "/docker/another": {0, 1}, + "/docker/somecontainer": {1, 0}, + }, + nil, + }, + { + ` +Sched Debug Version: v0.11, 3.13.0-29-generic #53-Ubuntu +runnable tasks: + task PID tree-key switches prio exec-runtime sum-exec sum-sleep +---------------------------------------------------------------------------------------------------------- + kthreadd 2 159918906.381680 397 120 159918906.381680 16.755308 4408983057.115372 0 / + kworker/0:0 9225 159645585.744016 8 120 159645585.744016 0.151033 9178573.870160 0 / + cadvisor 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/hash + someproc 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/somecontainer + +runnable tasks: + task PID tree-key switches prio exec-runtime sum-exec sum-sleep +---------------------------------------------------------------------------------------------------------- + kworker/0:0 9225 159645585.744016 8 120 159645585.744016 0.151033 9178573.870160 0 / + cadvisor 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/hash + other 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/another + +runnable tasks: + task PID tree-key switches prio exec-runtime sum-exec sum-sleep +---------------------------------------------------------------------------------------------------------- + kworker/0:0 9225 159645585.744016 8 120 159645585.744016 0.151033 9178573.870160 0 / + cadvisor 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/hash + other 15008 220535.926524 191029 120 220535.926524 3256.362378 672644.858661 0 /docker/another + + `, + map[string][]int{ + "/": {2, 1, 1}, + "/docker/hash": {1, 1, 1}, + "/docker/another": {0, 1, 1}, + "/docker/somecontainer": {1, 0, 0}, + }, + nil, + }, +} + +func TestSchedDebugReader(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + for _, testCase := range schedDebugToLoadsPerContainerPerCore { + mfs := mockfs.NewMockFileSystem(mockCtrl) + path := "/proc/sched_debug" + schedDebugContent := testCase.SchedDebugContent + mockfs.AddTextFile(mfs, path, schedDebugContent) + fs.ChangeFileSystem(mfs) + loads, err := NewSchedulerLoadReader() + if testCase.Error != nil { + if err == nil { + t.Fatal("expected error: %v", testCase.Error) + } + } + if err != nil { + t.Fatal(err) + } + containers, err := loads.AllContainers() + if err != nil { + t.Fatal(err) + } + if len(containers) != len(testCase.Loads) { + t.Errorf("expected %v container's information; received %v", len(testCase.Loads), len(containers)) + } + for _, container := range containers { + l, err := loads.Load(container) + if err != nil { + t.Fatal(err) + } + if expected, ok := testCase.Loads[container]; ok { + if !reflect.DeepEqual(expected, l) { + t.Errorf("Wrong load for container %v; should be %v, received %v\nsched_debug:\n%v\n", + container, expected, l, schedDebugContent) + continue + } + } else { + t.Errorf("unexpected container %v.\nsched_debug:\n%v\n", container, schedDebugContent) + } + } + } +}