cadvisor/utils/oomparser/oomparser.go

// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oomparser

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strconv"
	"time"

	"github.com/golang/glog"
	"github.com/google/cadvisor/utils"
)

var containerRegexp *regexp.Regexp = regexp.MustCompile(
	`Task in (.*) killed as a result of limit of (.*)`)
var lastLineRegexp *regexp.Regexp = regexp.MustCompile(
	`(^[A-Z]{1}[a-z]{2} .*[0-9]{1,2} [0-9]{1,2}:[0-9]{2}:[0-9]{2}) .* Killed process ([0-9]+) \(([0-9A-Za-z_]+)\)`)
var firstLineRegexp *regexp.Regexp = regexp.MustCompile(
	`invoked oom-killer:`)

// struct to hold file from which we obtain OomInstances
type OomParser struct {
	ioreader *bufio.Reader
}

// struct that contains information related to an OOM kill instance
type OomInstance struct {
	// process id of the killed process
	Pid int
	// the name of the killed process
	ProcessName string
	// the time that the process was reported to be killed,
	// accurate to the minute
	TimeOfDeath time.Time
	// the absolute name of the container that OOMed
	ContainerName string
	// the absolute name of the container that was killed
	// due to the OOM.
	VictimContainerName string
}

// gets the container name from a line and adds it to the oomInstance.
func getContainerName(line string, currentOomInstance *OomInstance) error {
	parsedLine := containerRegexp.FindStringSubmatch(line)
	if parsedLine == nil {
		return nil
	}
	currentOomInstance.ContainerName = path.Join("/", parsedLine[1])
	currentOomInstance.VictimContainerName = path.Join("/", parsedLine[2])
	return nil
}

// gets the pid, name, and date from a line and adds it to oomInstance
func getProcessNamePid(line string, currentOomInstance *OomInstance) (bool, error) {
	reList := lastLineRegexp.FindStringSubmatch(line)

	if reList == nil {
		return false, nil
	}
	const longForm = "Jan _2 15:04:05 2006"
	stringYear := strconv.Itoa(time.Now().Year())
	linetime, err := time.ParseInLocation(longForm, reList[1]+" "+stringYear, time.Local)
	if err != nil {
		return false, err
	}

	currentOomInstance.TimeOfDeath = linetime
	pid, err := strconv.Atoi(reList[2])
	if err != nil {
		return false, err
	}
	currentOomInstance.Pid = pid
	currentOomInstance.ProcessName = reList[3]
	return true, nil
}

// uses regex to see if line is the start of a kernel oom log
func checkIfStartOfOomMessages(line string) bool {
	potential_oom_start := firstLineRegexp.MatchString(line)
	if potential_oom_start {
		return true
	}
	return false
}

// reads the file and sends only complete lines over a channel to analyzeLines.
// Should prevent EOF errors that occur when lines are read before being fully
// written to the log. It reads line by line splitting on
// the "\n" character.
func readLinesFromFile(lineChannel chan string, ioreader *bufio.Reader) {
	linefragment := ""
	var line string
	var err error
	for true {
		line, err = ioreader.ReadString('\n')
		if err == io.EOF {
			if line != "" {
				linefragment += line
			}
			time.Sleep(100 * time.Millisecond)
		} else if err == nil {
			if linefragment != "" {
				line = linefragment + line
				linefragment = ""
			}
			lineChannel <- line
		} else if err != nil && err != io.EOF {
			glog.Errorf("exiting analyzeLinesHelper with error %v", err)
		}
	}
}

// Calls goroutine for readLinesFromFile, which feeds it complete lines.
// Lines are checked against a regexp to check for the pid, process name, etc.
// At the end of an oom message group, StreamOoms adds the new oomInstance to
// oomLog
func (self *OomParser) StreamOoms(outStream chan *OomInstance) {
	lineChannel := make(chan string, 10)
	go func() {
		readLinesFromFile(lineChannel, self.ioreader)
	}()

	for line := range lineChannel {
		in_oom_kernel_log := checkIfStartOfOomMessages(line)
		if in_oom_kernel_log {
			oomCurrentInstance := &OomInstance{
				ContainerName: "/",
			}
			finished := false
			for !finished {
				err := getContainerName(line, oomCurrentInstance)
				if err != nil {
					glog.Errorf("%v", err)
				}
				finished, err = getProcessNamePid(line, oomCurrentInstance)
				if err != nil {
					glog.Errorf("%v", err)
				}
				line = <-lineChannel
			}
			in_oom_kernel_log = false
			outStream <- oomCurrentInstance
		}
	}
	glog.Infof("exiting analyzeLines")
}

func callJournalctl() (io.ReadCloser, error) {
	cmd := exec.Command("journalctl", "-f")
	readcloser, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	return readcloser, err
}

func trySystemd() (*OomParser, error) {
	readcloser, err := callJournalctl()
	if err != nil {
		return nil, err
	}
	glog.V(1).Infof("oomparser using systemd")
	return &OomParser{
		ioreader: bufio.NewReader(readcloser),
	}, nil

}

// List of possible kernel log files. These are prioritized in order so that
// we will use the first one that is available.
var kernelLogFiles = []string{"/var/log/kern.log", "/var/log/messages", "/var/log/syslog"}

// looks for system files that contain kernel messages and if one is found, sets
// the systemFile attribute of the OomParser object
func getSystemFile() (string, error) {
	for _, logFile := range kernelLogFiles {
		if utils.FileExists(logFile) {
			glog.Infof("OOM parser using kernel log file: %q", logFile)
			return logFile, nil
		}
	}
	return "", fmt.Errorf("unable to find any kernel log file available from our set: %v", kernelLogFiles)
}

// initializes an OomParser object and calls getSystemFile to set the systemFile
// attribute.  Returns and OomParser object and an error
func New() (*OomParser, error) {
	systemFile, err := getSystemFile()
	if err != nil {
		return trySystemd()
	}
	file, err := os.Open(systemFile)
	if err != nil {
		return trySystemd()
	}
	return &OomParser{
		ioreader: bufio.NewReader(file),
	}, nil
}