cadvisor/utils/oomparser/oomparser.go

// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oomparser

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"path"
	"regexp"
	"strconv"
	"time"

	"github.com/golang/glog"
	"github.com/google/cadvisor/utils"
)

var containerRegexp *regexp.Regexp = regexp.MustCompile(
	`Task in (.*) killed as a result of limit of `)
var lastLineRegexp *regexp.Regexp = regexp.MustCompile(
	`(^[A-Z]{1}[a-z]{2} .*[0-9]{1,2} [0-9]{1,2}:[0-9]{2}:[0-9]{2}) .* Killed process ([0-9]+) \(([0-9A-Za-z_]+)\)`)
var firstLineRegexp *regexp.Regexp = regexp.MustCompile(
	`invoked oom-killer:`)

// struct to hold file from which we obtain OomInstances
type OomParser struct {
	systemFile string
}

// struct that contains information related to an OOM kill instance
type OomInstance struct {
	// process id of the killed process
	Pid int
	// the name of the killed process
	ProcessName string
	// the time that the process was reported to be killed,
	// accurate to the minute
	TimeOfDeath time.Time
	// the absolute name of the container that OOMed
	ContainerName string
}

// gets the container name from a line and adds it to the oomInstance.
func getContainerName(line string, currentOomInstance *OomInstance) error {
	parsedLine := containerRegexp.FindStringSubmatch(line)
	if parsedLine == nil {
		return nil
	}
	currentOomInstance.ContainerName = path.Join("/", parsedLine[1])
	return nil
}

// gets the pid, name, and date from a line and adds it to oomInstance
func getProcessNamePid(line string, currentOomInstance *OomInstance) (bool, error) {
	reList := lastLineRegexp.FindStringSubmatch(line)

	if reList == nil {
		return false, nil
	}
	const longForm = "Jan _2 15:04:05 2006"
	stringYear := strconv.Itoa(time.Now().Year())
	linetime, err := time.ParseInLocation(longForm, reList[1]+" "+stringYear, time.Local)
	if err != nil {
		return false, err
	}

	currentOomInstance.TimeOfDeath = linetime
	pid, err := strconv.Atoi(reList[2])
	if err != nil {
		return false, err
	}
	currentOomInstance.Pid = pid
	currentOomInstance.ProcessName = reList[3]
	return true, nil
}

// uses regex to see if line is the start of a kernel oom log
func checkIfStartOfOomMessages(line string) bool {
	potential_oom_start := firstLineRegexp.MatchString(line)
	if potential_oom_start {
		return true
	}
	return false
}

// reads the file and sends only complete lines over a channel to analyzeLines.
// Should prevent EOF errors that occur when lines are read before being fully
// written to the log. It reads line by line splitting on
// the "\n" character.
func readLinesFromFile(lineChannel chan string, ioreader *bufio.Reader) {
	linefragment := ""
	var line string
	var err error
	for true {
		line, err = ioreader.ReadString('\n')
		if err == io.EOF {
			if line != "" {
				linefragment += line
			}
			time.Sleep(100 * time.Millisecond)
		} else if err == nil {
			if linefragment != "" {
				line = linefragment + line
				linefragment = ""
			}
			lineChannel <- line
		} else if err != nil && err != io.EOF {
			glog.Errorf("exiting analyzeLinesHelper with error %v", err)
		}
	}
}

// Calls goroutine for analyzeLinesHelper, which feeds it complete lines.
// Lines are checked against a regexp to check for the pid, process name, etc.
// At the end of an oom message group, AnalyzeLines adds the new oomInstance to
// oomLog
func (self *OomParser) analyzeLines(ioreader *bufio.Reader, outStream chan *OomInstance) {
	lineChannel := make(chan string, 10)
	go func() {
		readLinesFromFile(lineChannel, ioreader)
	}()

	for line := range lineChannel {
		in_oom_kernel_log := checkIfStartOfOomMessages(line)

		if in_oom_kernel_log {
			oomCurrentInstance := &OomInstance{
				ContainerName: "/",
			}
			finished := false
			for !finished {
				err := getContainerName(line, oomCurrentInstance)
				if err != nil {
					glog.Errorf("%v", err)
				}
				finished, err = getProcessNamePid(line, oomCurrentInstance)
				if err != nil {
					glog.Errorf("%v", err)
				}
				line = <-lineChannel
			}
			in_oom_kernel_log = false
			outStream <- oomCurrentInstance
		}
	}
	glog.Infof("exiting analyzeLines")
}

// looks for system files that contain kernel messages and if one is found, sets
// the systemFile attribute of the OomParser object
func getSystemFile() (string, error) {
	const varLogMessages = "/var/log/messages"
	const varLogSyslog = "/var/log/syslog"
	if utils.FileExists(varLogMessages) {
		return varLogMessages, nil
	} else if utils.FileExists(varLogSyslog) {
		return varLogSyslog, nil
	}
	return "", fmt.Errorf("neither %s nor %s exists from which to read kernel errors", varLogMessages, varLogSyslog)
}

// calls a go routine that populates self.OomInstances and fills the argument
// channel with OomInstance objects as they are read from the file.
// opens the OomParser's systemFile which was set in getSystemFile
// to look for OOM messages by calling AnalyzeLines.  Takes in the argument
// outStream, which is passed in by the user and passed to AnalyzeLines.
// OomInstance objects are added to outStream when they are found by
// AnalyzeLines
func (self *OomParser) StreamOoms(outStream chan *OomInstance) error {
	file, err := os.Open(self.systemFile)
	if err != nil {
		return err
	}
	ioreader := bufio.NewReader(file)

	// Process the events received from the kernel.
	go func() {
		self.analyzeLines(ioreader, outStream)
	}()
	return nil
}

// initializes an OomParser object and calls getSystemFile to set the systemFile
// attribute.  Returns and OomParser object and an error
func New() (*OomParser, error) {
	systemFileName, err := getSystemFile()
	if err != nil {
		return nil, err
	}
	return &OomParser{
		systemFile: systemFileName}, nil
}