prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

oomparser: don't get stuck for certain processes

Browse Source 
The oomparser logic would end up stuck, unable to detect the end of a
given oom trace, for any process with a name that didn't match \w+.

This includes processes like 'python3.4' due to the '.', or
'docker-containerd' due to the '-'.

This fix was included in pr #1544 last year, but since that PR seems
dead it seems like a good idea to break this more important fix out.

I've updated the tests such that they would have caught this issue.
This commit is contained in:
Euan Kemp 2017-08-01 13:32:20 -07:00
parent ee7e1dd52b
commit fe722c7b02
2 changed files with 15 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
utils/oomparser/oomparser.go
View File

@ -32,7 +32,7 @@ import (
var ( var (
containerRegexp = regexp.MustCompile(`Task in (.*) killed as a result of limit of (.*)`) containerRegexp = regexp.MustCompile(`Task in (.*) killed as a result of limit of (.*)`)
lastLineRegexp = regexp.MustCompile(`(^[A-Z][a-z]{2} .*[0-9]{1,2} [0-9]{1,2}:[0-9]{2}:[0-9]{2}) .* Killed process ([0-9]+) \(([\w]+)\)`) lastLineRegexp = regexp.MustCompile(`(^[A-Z][a-z]{2} .*[0-9]{1,2} [0-9]{1,2}:[0-9]{2}:[0-9]{2}) .* Killed process ([0-9]+) \((.+)\)`)
firstLineRegexp = regexp.MustCompile(`invoked oom-killer:`) firstLineRegexp = regexp.MustCompile(`invoked oom-killer:`)
) )

17
utils/oomparser/oomparser_test.go
View File

@ -26,7 +26,7 @@ import (
) )
const startLine = "Jan 21 22:01:49 localhost kernel: [62278.816267] ruby invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0" const startLine = "Jan 21 22:01:49 localhost kernel: [62278.816267] ruby invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0"
const endLine = "Jan 21 22:01:49 localhost kernel: [62279.421192] Killed process 19667 (evilprogram2) total-vm:1460016kB, anon-rss:1414008kB, file-rss:4kB" const endLine = "Jan 21 22:01:49 localhost kernel: [62279.421192] Killed process 19667 (evil-program2) total-vm:1460016kB, anon-rss:1414008kB, file-rss:4kB"
const containerLine = "Jan 26 14:10:07 kateknister0.mtv.corp.google.com kernel: [1814368.465205] Task in /mem2 killed as a result of limit of /mem3" const containerLine = "Jan 26 14:10:07 kateknister0.mtv.corp.google.com kernel: [1814368.465205] Task in /mem2 killed as a result of limit of /mem3"
const containerLogFile = "containerOomExampleLog.txt" const containerLogFile = "containerOomExampleLog.txt"
const systemLogFile = "systemOomExampleLog.txt" const systemLogFile = "systemOomExampleLog.txt"
@ -104,8 +104,8 @@ func TestGetProcessNamePid(t *testing.T) {
if !couldParseLine { if !couldParseLine {
t.Errorf("good line fed to getProcessNamePid should return true but returned %v", couldParseLine) t.Errorf("good line fed to getProcessNamePid should return true but returned %v", couldParseLine)
} }
if currentOomInstance.ProcessName != "evilprogram2" { if currentOomInstance.ProcessName != "evil-program2" {
t.Errorf("getProcessNamePid should have set processName to evilprogram2, not %s", currentOomInstance.ProcessName) t.Errorf("getProcessNamePid should have set processName to evil-program2, not %s", currentOomInstance.ProcessName)
} }
if currentOomInstance.Pid != 19667 { if currentOomInstance.Pid != 19667 {
t.Errorf("getProcessNamePid should have set PID to 19667, not %d", currentOomInstance.Pid) t.Errorf("getProcessNamePid should have set PID to 19667, not %d", currentOomInstance.Pid)
@ -126,6 +126,17 @@ func TestCheckIfStartOfMessages(t *testing.T) {
} }
} }
func TestLastLineRegex(t *testing.T) {
processNames := []string{"foo", "python3.4", "foo-bar", "Plex Media Server", "x86_64-pc-linux-gnu-c++-5.4.0", "[", "()", `"with quotes"`}
for _, name := range processNames {
line := fmt.Sprintf("Jan 21 22:01:49 localhost kernel: [62279.421192] Killed process 1234 (%s) total-vm:1460016kB, anon-rss:1414008kB, file-rss:4kB", name)
oomInfo := &OomInstance{}
getProcessNamePid(line, oomInfo)
assert.Equal(t, 1234, oomInfo.Pid)
assert.Equal(t, name, oomInfo.ProcessName)
}
}
func TestStreamOomsContainer(t *testing.T) { func TestStreamOomsContainer(t *testing.T) {
expectedContainerOomInstance := createExpectedContainerOomInstance(t) expectedContainerOomInstance := createExpectedContainerOomInstance(t)
helpTestStreamOoms(expectedContainerOomInstance, containerLogFile, t) helpTestStreamOoms(expectedContainerOomInstance, containerLogFile, t)