Detect race conditions in jenkins integration tests

Injected a race into cadvisor.go and verified this caught it. Also, fix
the runner to grab the logs even when the tests don't complete.

build/jenkins_e2e.sh

@@ -19,6 +19,9 @@ set -x
 
 BUILDER=${BUILDER:-false} # Whether this is running a PR builder job.
 
+export GO_FLAGS="-race"
+export GORACE="halt_on_error=1"
+
 go get -u github.com/tools/godep
 ./build/presubmit.sh
 godep go build -tags test github.com/google/cadvisor/integration/runner

integration/runner/runner.go

@@ -88,7 +88,7 @@ func RunSshCommand(cmd string, args ...string) error {
 	return RunCommand(cmd, args...)
 }
 
-func PushAndRunTests(host, testDir string) error {
+func PushAndRunTests(host, testDir string) (result error) {
 	// Push binary.
 	glog.Infof("Pushing cAdvisor binary to %q...", host)
 
@@ -113,7 +113,7 @@ func PushAndRunTests(host, testDir string) error {
 	portStr := strconv.Itoa(*port)
 	errChan := make(chan error)
 	go func() {
-		err = RunSshCommand("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr --docker_env_metadata_whitelist=TEST_VAR  &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
+		err = RunSshCommand("ssh", host, "--", fmt.Sprintf("sudo GORACE='halt_on_error=1' %s --port %s --logtostderr --docker_env_metadata_whitelist=TEST_VAR  &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
 		if err != nil {
 			errChan <- fmt.Errorf("error running cAdvisor: %v", err)
 		}
@@ -124,6 +124,30 @@ func PushAndRunTests(host, testDir string) error {
 			glog.Errorf("Failed to cleanup: %v", err)
 		}
 	}()
+	defer func() {
+		if result != nil {
+			// Copy logs from the host
+			err := RunSshCommand("scp", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
+			if err != nil {
+				result = fmt.Errorf("error fetching logs: %v for %v", err, result)
+				return
+			}
+			defer os.Remove("./log.txt")
+			logs, err := ioutil.ReadFile("./log.txt")
+			if err != nil {
+				result = fmt.Errorf("error reading local log file: %v for %v", err, result)
+				return
+			}
+			glog.Errorf("----------------------\nLogs from Host: %q\n%v\n", host, string(logs))
+
+			// Get attributes for debugging purposes.
+			attributes, err := getAttributes(host, portStr)
+			if err != nil {
+				glog.Errorf("Failed to read host attributes: %v", err)
+			}
+			result = fmt.Errorf("error on host %s: %v\n%+v", host, result, attributes)
+		}
+	}()
 
 	// Wait for cAdvisor to come up.
 	endTime := time.Now().Add(*cadvisorTimeout)
@@ -146,11 +170,6 @@ func PushAndRunTests(host, testDir string) error {
 		return fmt.Errorf("timed out waiting for cAdvisor to come up at host %q", host)
 	}
 
-	// Get attributes for debugging purposes.
-	attributes, err := getAttributes(host, portStr)
-	if err != nil {
-		return fmt.Errorf("%v - %q", err, host)
-	}
 	// Run the tests in a retry loop.
 	glog.Infof("Running integration tests targeting %q...", host)
 	for i := 0; i <= *testRetryCount; i++ {
@@ -173,21 +192,6 @@ func PushAndRunTests(host, testDir string) error {
 			break
 		}
 	}
-	if err != nil {
-		// Copy logs from the host
-		// Declare new error or it will get shadowed by logs, err := <>  and we won't be able to unset it from nil
-		err2 := RunSshCommand("scp", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
-		if err2 != nil {
-			return fmt.Errorf("error fetching logs: %v for %v", err2, err)
-		}
-		defer os.Remove("./log.txt")
-		logs, err2 := ioutil.ReadFile("./log.txt")
-		if err2 != nil {
-			return fmt.Errorf("error reading local log file: %v for %v", err2, err)
-		}
-		glog.Errorf("----------------------\nLogs from Host: %q\n%v\n", host, string(logs))
-		err = fmt.Errorf("error on host %s: %v\n%+v", host, err, attributes)
-	}
 	return err
 }
 
@@ -204,7 +208,7 @@ func Run() error {
 
 	// Build cAdvisor.
 	glog.Infof("Building cAdvisor...")
-	err := RunCommand("godep", "go", "build", "github.com/google/cadvisor")
+	err := RunCommand("build/build.sh")
 	if err != nil {
 		return err
 	}