Merge pull request #1108 from pwittrock/integration-test-use-direct-ssh

Use vanilla ssh instead of gcloud ssh for e2e tests

integration/common/gce.go

@@ -1,81 +0,0 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package common
-
-import (
-	"flag"
-	"fmt"
-	"os/exec"
-	"regexp"
-
-	"google.golang.org/cloud/compute/metadata"
-)
-
-var zone = flag.String("zone", "us-central1-f", "Zone the instances are running in")
-var project = flag.String("project", "", "Project the instances are running in")
-
-var gceInternalIpRegexp = regexp.MustCompile(`\s+networkIP:\s+([0-9.:]+)\n`)
-var gceExternalIpRegexp = regexp.MustCompile(`\s+natIP:\s+([0-9.:]+)\n`)
-
-// Gets the IP of the specified GCE instance.
-func GetGceIp(hostname string) (string, error) {
-	if hostname == "localhost" {
-		return "127.0.0.1", nil
-	}
-
-	args := []string{"compute"}
-	args = append(args, getProjectFlag()...)
-	args = append(args, "instances", "describe")
-	args = append(args, getZoneFlag()...)
-	args = append(args, hostname)
-	out, err := exec.Command("gcloud", args...).CombinedOutput()
-	if err != nil {
-		return "", fmt.Errorf("failed to get instance information for %q with error %v and output %s", hostname, err, string(out))
-	}
-
-	// Use the internal IP within GCE and the external one outside.
-	var matches []string
-	if metadata.OnGCE() {
-		matches = gceInternalIpRegexp.FindStringSubmatch(string(out))
-	} else {
-		matches = gceExternalIpRegexp.FindStringSubmatch(string(out))
-	}
-	if len(matches) == 0 {
-		return "", fmt.Errorf("failed to find IP from output %q", string(out))
-	}
-	return matches[1], nil
-}
-
-func getZoneFlag() []string {
-	if *zone == "" {
-		return []string{}
-	}
-	return []string{"--zone", *zone}
-}
-
-func getProjectFlag() []string {
-	if *project == "" {
-		return []string{}
-	}
-	return []string{"--project", *project}
-}
-func GetGCComputeArgs(cmd string, cmdArgs ...string) []string {
-	args := []string{"compute"}
-	args = append(args, getProjectFlag()...)
-	args = append(args, cmd)
-	args = append(args, getZoneFlag()...)
-	args = append(args, cmdArgs...)
-	return args
-}

integration/framework/framework.go

@@ -25,11 +25,11 @@ import (
 
 	"github.com/google/cadvisor/client"
 	"github.com/google/cadvisor/client/v2"
-	"github.com/google/cadvisor/integration/common"
 )
 
 var host = flag.String("host", "localhost", "Address of the host being tested")
 var port = flag.Int("port", 8080, "Port of the application on the host being tested")
+var sshOptions = flag.String("ssh-options", "", "Command line options for ssh")
 
 // Integration test framework.
 type Framework interface {
@@ -69,21 +69,10 @@ func New(t *testing.T) Framework {
 	}
 
 	// Try to see if non-localhost hosts are GCE instances.
-	var gceInstanceName string
-	hostname := *host
-	if hostname != "localhost" {
-		gceInstanceName = hostname
-		gceIp, err := common.GetGceIp(hostname)
-		if err == nil {
-			hostname = gceIp
-		}
-	}
-
 	fm := &realFramework{
 		hostname: HostnameInfo{
-			Host:            hostname,
+			Host: *host,
-			Port:            *port,
+			Port: *port,
-			GceInstanceName: gceInstanceName,
 		},
 		t:        t,
 		cleanups: make([]func(), 0),
@@ -159,9 +148,8 @@ type dockerActions struct {
 }
 
 type HostnameInfo struct {
-	Host            string
+	Host string
-	Port            int
+	Port int
-	GceInstanceName string
 }
 
 // Returns: http://<host>:<port>/
@@ -328,8 +316,11 @@ func (self shellActions) Run(command string, args ...string) (string, string) {
 		cmd = exec.Command(command, args...)
 	} else {
 		// We must SSH to the remote machine and run the command.
-		args = append(common.GetGCComputeArgs("ssh", self.fm.Hostname().GceInstanceName, "--", command), args...)
+		args = append([]string{self.fm.Hostname().Host, "--", command}, args...)
-		cmd = exec.Command("gcloud", args...)
+		if *sshOptions != "" {
+			args = append(strings.Split(*sshOptions, " "), args...)
+		}
+		cmd = exec.Command("ssh", args...)
 	}
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
@@ -350,8 +341,11 @@ func (self shellActions) RunStress(command string, args ...string) (string, stri
 		cmd = exec.Command(command, args...)
 	} else {
 		// We must SSH to the remote machine and run the command.
-		args = append(common.GetGCComputeArgs("ssh", self.fm.Hostname().GceInstanceName, "--", command), args...)
+		args = append([]string{self.fm.Hostname().Host, "--", command}, args...)
-		cmd = exec.Command("gcloud", args...)
+		if *sshOptions != "" {
+			args = append(strings.Split(*sshOptions, " "), args...)
+		}
+		cmd = exec.Command("ssh", args...)
 	}
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer

integration/runner/runner.go

@@ -32,20 +32,20 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/cadvisor/integration/common"
-
-	cadvisorApi "github.com/google/cadvisor/info/v2"
-
 	"github.com/golang/glog"
+	cadvisorApi "github.com/google/cadvisor/info/v2"
 )
 
+// must be able to ssh into hosts without password
+// godep go run ./integration/runner/runner.go --logtostderr --v 2 --ssh-config <.ssh/config file> <list of hosts>
+
 const cadvisorBinary = "cadvisor"
 
 var cadvisorTimeout = flag.Duration("cadvisor_timeout", 15*time.Second, "Time to wait for cAdvisor to come up on the remote host")
 var port = flag.Int("port", 8080, "Port in which to start cAdvisor in the remote host")
 var testRetryCount = flag.Int("test-retry-count", 3, "Number of times to retry failed tests before failing.")
 var testRetryWhitelist = flag.String("test-retry-whitelist", "", "Path to newline separated list of regexexp for test failures that should be retried.  If empty, no tests are retried.")
-var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.") // Used in a follow up
+var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.")
 var retryRegex *regexp.Regexp
 
 func getAttributes(ipAddress, portStr string) (*cadvisorApi.Attributes, error) {
@@ -78,23 +78,29 @@ func RunCommand(cmd string, args ...string) error {
 	return nil
 }
 
+func RunSshCommand(cmd string, args ...string) error {
+	if *sshOptions != "" {
+		args = append(strings.Split(*sshOptions, " "), args...)
+	}
+	return RunCommand(cmd, args...)
+}
+
 func PushAndRunTests(host, testDir string) error {
 	// Push binary.
 	glog.Infof("Pushing cAdvisor binary to %q...", host)
-	args := common.GetGCComputeArgs("ssh", host, "--", "mkdir", "-p", testDir)
+
-	err := RunCommand("gcloud", args...)
+	err := RunSshCommand("ssh", host, "--", "mkdir", "-p", testDir)
 	if err != nil {
 		return fmt.Errorf("failed to make remote testing directory: %v", err)
 	}
 	defer func() {
-		args := common.GetGCComputeArgs("ssh", host, "--", "rm", "-rf", testDir)
+		err = RunSshCommand("ssh", host, "--", "rm", "-rf", testDir)
-		err := RunCommand("gcloud", args...)
 		if err != nil {
 			glog.Errorf("Failed to cleanup test directory: %v", err)
 		}
 	}()
-	args = common.GetGCComputeArgs("copy-files", cadvisorBinary, fmt.Sprintf("%s:%s", host, testDir))
+
-	err = RunCommand("gcloud", args...)
+	err = RunSshCommand("scp", "-r", cadvisorBinary, fmt.Sprintf("%s:%s", host, testDir))
 	if err != nil {
 		return fmt.Errorf("failed to copy binary: %v", err)
 	}
@@ -104,25 +110,18 @@ func PushAndRunTests(host, testDir string) error {
 	portStr := strconv.Itoa(*port)
 	errChan := make(chan error)
 	go func() {
-		args = common.GetGCComputeArgs("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
+		err = RunSshCommand("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr  &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
-		err = RunCommand("gcloud", args...)
 		if err != nil {
 			errChan <- fmt.Errorf("error running cAdvisor: %v", err)
 		}
 	}()
 	defer func() {
-		args = common.GetGCComputeArgs("ssh", host, "--", "sudo", "pkill", cadvisorBinary)
+		err = RunSshCommand("ssh", host, "--", "sudo", "pkill", cadvisorBinary)
-		err := RunCommand("gcloud", args...)
 		if err != nil {
 			glog.Errorf("Failed to cleanup: %v", err)
 		}
 	}()
 
-	ipAddress, err := common.GetGceIp(host)
-	if err != nil {
-		return fmt.Errorf("failed to get GCE IP: %v", err)
-	}
-
 	// Wait for cAdvisor to come up.
 	endTime := time.Now().Add(*cadvisorTimeout)
 	done := false
@@ -133,7 +132,7 @@ func PushAndRunTests(host, testDir string) error {
 			return err
 		case <-time.After(500 * time.Millisecond):
 			// Stop waiting when cAdvisor is healthy..
-			resp, err := http.Get(fmt.Sprintf("http://%s:%s/healthz", ipAddress, portStr))
+			resp, err := http.Get(fmt.Sprintf("http://%s:%s/healthz", host, portStr))
 			if err == nil && resp.StatusCode == http.StatusOK {
 				done = true
 				break
@@ -145,7 +144,7 @@ func PushAndRunTests(host, testDir string) error {
 	}
 
 	// Get attributes for debugging purposes.
-	attributes, err := getAttributes(ipAddress, portStr)
+	attributes, err := getAttributes(host, portStr)
 	if err != nil {
 		return fmt.Errorf("%v - %q", err, host)
 	}
@@ -158,7 +157,8 @@ func PushAndRunTests(host, testDir string) error {
 			glog.Warningf("Retrying (%d of %d) tests on host %s due to error %v", i, *testRetryCount, host, err)
 		}
 		// Run the command
-		err = RunCommand("godep", "go", "test", "github.com/google/cadvisor/integration/tests/...", "--host", host, "--port", portStr)
+
+		err = RunCommand("godep", "go", "test", "github.com/google/cadvisor/integration/tests/...", "--host", host, "--port", portStr, "--ssh-options", *sshOptions)
 		if err == nil {
 			// On success, break out of retry loop
 			break
@@ -172,9 +172,8 @@ func PushAndRunTests(host, testDir string) error {
 	}
 	if err != nil {
 		// Copy logs from the host
-		args = common.GetGCComputeArgs("copy-files", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
 		// Declare new error or it will get shadowed by logs, err := <>  and we won't be able to unset it from nil
-		err2 := RunCommand("gcloud", args...)
+		err2 := RunSshCommand("scp", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
 		if err2 != nil {
 			return fmt.Errorf("error fetching logs: %v for %v", err2, err)
 		}