prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Detect race conditions in jenkins integration tests

Browse Source 
Injected a race into cadvisor.go and verified this caught it. Also, fix
the runner to grab the logs even when the tests don't complete.
This commit is contained in:
Tim St. Clair 2016-06-30 19:10:08 -07:00
parent e6c30f7e8d
commit 2c69c0b634
2 changed files with 30 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
build/jenkins_e2e.sh
View File

@ -19,6 +19,9 @@ set -x
BUILDER=${BUILDER:-false} # Whether this is running a PR builder job.
export GO_FLAGS="-race"
export GORACE="halt_on_error=1"
go get -u github.com/tools/godep
./build/presubmit.sh
godep go build -tags test github.com/google/cadvisor/integration/runner

50
integration/runner/runner.go
View File

@ -88,7 +88,7 @@ func RunSshCommand(cmd string, args ...string) error {
return RunCommand(cmd, args...)
}
func PushAndRunTests(host, testDir string) error {
func PushAndRunTests(host, testDir string) (result error) {
// Push binary.
glog.Infof("Pushing cAdvisor binary to %q...", host)
@ -113,7 +113,7 @@ func PushAndRunTests(host, testDir string) error {
portStr := strconv.Itoa(*port)
errChan := make(chan error)
go func() {
err = RunSshCommand("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr --docker_env_metadata_whitelist=TEST_VAR &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
err = RunSshCommand("ssh", host, "--", fmt.Sprintf("sudo GORACE='halt_on_error=1' %s --port %s --logtostderr --docker_env_metadata_whitelist=TEST_VAR &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
if err != nil {
errChan <- fmt.Errorf("error running cAdvisor: %v", err)
}
@ -124,6 +124,30 @@ func PushAndRunTests(host, testDir string) error {
glog.Errorf("Failed to cleanup: %v", err)
}
}()
defer func() {
if result != nil {
// Copy logs from the host
err := RunSshCommand("scp", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
if err != nil {
result = fmt.Errorf("error fetching logs: %v for %v", err, result)
return
}
defer os.Remove("./log.txt")
logs, err := ioutil.ReadFile("./log.txt")
if err != nil {
result = fmt.Errorf("error reading local log file: %v for %v", err, result)
return
}
glog.Errorf("----------------------\nLogs from Host: %q\n%v\n", host, string(logs))
// Get attributes for debugging purposes.
attributes, err := getAttributes(host, portStr)
if err != nil {
glog.Errorf("Failed to read host attributes: %v", err)
}
result = fmt.Errorf("error on host %s: %v\n%+v", host, result, attributes)
}
}()
// Wait for cAdvisor to come up.
endTime := time.Now().Add(*cadvisorTimeout)
@ -146,11 +170,6 @@ func PushAndRunTests(host, testDir string) error {
return fmt.Errorf("timed out waiting for cAdvisor to come up at host %q", host)
}
// Get attributes for debugging purposes.
attributes, err := getAttributes(host, portStr)
if err != nil {
return fmt.Errorf("%v - %q", err, host)
}
// Run the tests in a retry loop.
glog.Infof("Running integration tests targeting %q...", host)
for i := 0; i <= *testRetryCount; i++ {
@ -173,21 +192,6 @@ func PushAndRunTests(host, testDir string) error {
break
}
}
if err != nil {
// Copy logs from the host
// Declare new error or it will get shadowed by logs, err := <> and we won't be able to unset it from nil
err2 := RunSshCommand("scp", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
if err2 != nil {
return fmt.Errorf("error fetching logs: %v for %v", err2, err)
}
defer os.Remove("./log.txt")
logs, err2 := ioutil.ReadFile("./log.txt")
if err2 != nil {
return fmt.Errorf("error reading local log file: %v for %v", err2, err)
}
glog.Errorf("----------------------\nLogs from Host: %q\n%v\n", host, string(logs))
err = fmt.Errorf("error on host %s: %v\n%+v", host, err, attributes)
}
return err
}
@ -204,7 +208,7 @@ func Run() error {
// Build cAdvisor.
glog.Infof("Building cAdvisor...")
err := RunCommand("godep", "go", "build", "github.com/google/cadvisor")
err := RunCommand("build/build.sh")
if err != nil {
return err
}