prometheus/cadvisor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #1108 from pwittrock/integration-test-use-direct-ssh

Browse Source 
Use vanilla ssh instead of gcloud ssh for e2e tests
This commit is contained in:
Phillip Wittrock 2016-02-11 17:30:40 -08:00
commit 9baa6b5def
3 changed files with 39 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
integration/common/gce.go
View File

@ -1,81 +0,0 @@
// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package common
import (
"flag"
"fmt"
"os/exec"
"regexp"
"google.golang.org/cloud/compute/metadata"
)
var zone = flag.String("zone", "us-central1-f", "Zone the instances are running in")
var project = flag.String("project", "", "Project the instances are running in")
var gceInternalIpRegexp = regexp.MustCompile(`\s+networkIP:\s+([0-9.:]+)\n`)
var gceExternalIpRegexp = regexp.MustCompile(`\s+natIP:\s+([0-9.:]+)\n`)
// Gets the IP of the specified GCE instance.
func GetGceIp(hostname string) (string, error) {
if hostname == "localhost" {
return "127.0.0.1", nil
}
args := []string{"compute"}
args = append(args, getProjectFlag()...)
args = append(args, "instances", "describe")
args = append(args, getZoneFlag()...)
args = append(args, hostname)
out, err := exec.Command("gcloud", args...).CombinedOutput()
if err != nil {
return "", fmt.Errorf("failed to get instance information for %q with error %v and output %s", hostname, err, string(out))
}
// Use the internal IP within GCE and the external one outside.
var matches []string
if metadata.OnGCE() {
matches = gceInternalIpRegexp.FindStringSubmatch(string(out))
} else {
matches = gceExternalIpRegexp.FindStringSubmatch(string(out))
}
if len(matches) == 0 {
return "", fmt.Errorf("failed to find IP from output %q", string(out))
}
return matches[1], nil
}
func getZoneFlag() []string {
if *zone == "" {
return []string{}
}
return []string{"--zone", *zone}
}
func getProjectFlag() []string {
if *project == "" {
return []string{}
}
return []string{"--project", *project}
}
func GetGCComputeArgs(cmd string, cmdArgs ...string) []string {
args := []string{"compute"}
args = append(args, getProjectFlag()...)
args = append(args, cmd)
args = append(args, getZoneFlag()...)
args = append(args, cmdArgs...)
return args
}

36
integration/framework/framework.go
View File

@ -25,11 +25,11 @@ import (
"github.com/google/cadvisor/client"
"github.com/google/cadvisor/client/v2"
"github.com/google/cadvisor/integration/common"
)
var host = flag.String("host", "localhost", "Address of the host being tested")
var port = flag.Int("port", 8080, "Port of the application on the host being tested")
var sshOptions = flag.String("ssh-options", "", "Command line options for ssh")
// Integration test framework.
type Framework interface {
@ -69,21 +69,10 @@ func New(t *testing.T) Framework {
}
// Try to see if non-localhost hosts are GCE instances.
var gceInstanceName string
hostname := *host
if hostname != "localhost" {
gceInstanceName = hostname
gceIp, err := common.GetGceIp(hostname)
if err == nil {
hostname = gceIp
}
}
fm := &realFramework{
hostname: HostnameInfo{
Host: hostname,
Port: *port,
GceInstanceName: gceInstanceName,
Host: *host,
Port: *port,
},
t: t,
cleanups: make([]func(), 0),
@ -159,9 +148,8 @@ type dockerActions struct {
}
type HostnameInfo struct {
Host string
Port int
GceInstanceName string
Host string
Port int
}
// Returns: http://<host>:<port>/
@ -328,8 +316,11 @@ func (self shellActions) Run(command string, args ...string) (string, string) {
cmd = exec.Command(command, args...)
} else {
// We must SSH to the remote machine and run the command.
args = append(common.GetGCComputeArgs("ssh", self.fm.Hostname().GceInstanceName, "--", command), args...)
cmd = exec.Command("gcloud", args...)
args = append([]string{self.fm.Hostname().Host, "--", command}, args...)
if *sshOptions != "" {
args = append(strings.Split(*sshOptions, " "), args...)
}
cmd = exec.Command("ssh", args...)
}
var stdout bytes.Buffer
var stderr bytes.Buffer
@ -350,8 +341,11 @@ func (self shellActions) RunStress(command string, args ...string) (string, stri
cmd = exec.Command(command, args...)
} else {
// We must SSH to the remote machine and run the command.
args = append(common.GetGCComputeArgs("ssh", self.fm.Hostname().GceInstanceName, "--", command), args...)
cmd = exec.Command("gcloud", args...)
args = append([]string{self.fm.Hostname().Host, "--", command}, args...)
if *sshOptions != "" {
args = append(strings.Split(*sshOptions, " "), args...)
}
cmd = exec.Command("ssh", args...)
}
var stdout bytes.Buffer
var stderr bytes.Buffer

49
integration/runner/runner.go
View File

@ -32,20 +32,20 @@ import (
"sync"
"time"
"github.com/google/cadvisor/integration/common"
cadvisorApi "github.com/google/cadvisor/info/v2"
"github.com/golang/glog"
cadvisorApi "github.com/google/cadvisor/info/v2"
)
// must be able to ssh into hosts without password
// godep go run ./integration/runner/runner.go --logtostderr --v 2 --ssh-config <.ssh/config file> <list of hosts>
const cadvisorBinary = "cadvisor"
var cadvisorTimeout = flag.Duration("cadvisor_timeout", 15*time.Second, "Time to wait for cAdvisor to come up on the remote host")
var port = flag.Int("port", 8080, "Port in which to start cAdvisor in the remote host")
var testRetryCount = flag.Int("test-retry-count", 3, "Number of times to retry failed tests before failing.")
var testRetryWhitelist = flag.String("test-retry-whitelist", "", "Path to newline separated list of regexexp for test failures that should be retried. If empty, no tests are retried.")
var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.") // Used in a follow up
var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.")
var retryRegex *regexp.Regexp
func getAttributes(ipAddress, portStr string) (*cadvisorApi.Attributes, error) {
@ -78,23 +78,29 @@ func RunCommand(cmd string, args ...string) error {
return nil
}
func RunSshCommand(cmd string, args ...string) error {
if *sshOptions != "" {
args = append(strings.Split(*sshOptions, " "), args...)
}
return RunCommand(cmd, args...)
}
func PushAndRunTests(host, testDir string) error {
// Push binary.
glog.Infof("Pushing cAdvisor binary to %q...", host)
args := common.GetGCComputeArgs("ssh", host, "--", "mkdir", "-p", testDir)
err := RunCommand("gcloud", args...)
err := RunSshCommand("ssh", host, "--", "mkdir", "-p", testDir)
if err != nil {
return fmt.Errorf("failed to make remote testing directory: %v", err)
}
defer func() {
args := common.GetGCComputeArgs("ssh", host, "--", "rm", "-rf", testDir)
err := RunCommand("gcloud", args...)
err = RunSshCommand("ssh", host, "--", "rm", "-rf", testDir)
if err != nil {
glog.Errorf("Failed to cleanup test directory: %v", err)
}
}()
args = common.GetGCComputeArgs("copy-files", cadvisorBinary, fmt.Sprintf("%s:%s", host, testDir))
err = RunCommand("gcloud", args...)
err = RunSshCommand("scp", "-r", cadvisorBinary, fmt.Sprintf("%s:%s", host, testDir))
if err != nil {
return fmt.Errorf("failed to copy binary: %v", err)
}
@ -104,25 +110,18 @@ func PushAndRunTests(host, testDir string) error {
portStr := strconv.Itoa(*port)
errChan := make(chan error)
go func() {
args = common.GetGCComputeArgs("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
err = RunCommand("gcloud", args...)
err = RunSshCommand("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
if err != nil {
errChan <- fmt.Errorf("error running cAdvisor: %v", err)
}
}()
defer func() {
args = common.GetGCComputeArgs("ssh", host, "--", "sudo", "pkill", cadvisorBinary)
err := RunCommand("gcloud", args...)
err = RunSshCommand("ssh", host, "--", "sudo", "pkill", cadvisorBinary)
if err != nil {
glog.Errorf("Failed to cleanup: %v", err)
}
}()
ipAddress, err := common.GetGceIp(host)
if err != nil {
return fmt.Errorf("failed to get GCE IP: %v", err)
}
// Wait for cAdvisor to come up.
endTime := time.Now().Add(*cadvisorTimeout)
done := false
@ -133,7 +132,7 @@ func PushAndRunTests(host, testDir string) error {
return err
case <-time.After(500 * time.Millisecond):
// Stop waiting when cAdvisor is healthy..
resp, err := http.Get(fmt.Sprintf("http://%s:%s/healthz", ipAddress, portStr))
resp, err := http.Get(fmt.Sprintf("http://%s:%s/healthz", host, portStr))
if err == nil && resp.StatusCode == http.StatusOK {
done = true
break
@ -145,7 +144,7 @@ func PushAndRunTests(host, testDir string) error {
}
// Get attributes for debugging purposes.
attributes, err := getAttributes(ipAddress, portStr)
attributes, err := getAttributes(host, portStr)
if err != nil {
return fmt.Errorf("%v - %q", err, host)
}
@ -158,7 +157,8 @@ func PushAndRunTests(host, testDir string) error {
glog.Warningf("Retrying (%d of %d) tests on host %s due to error %v", i, *testRetryCount, host, err)
}
// Run the command
err = RunCommand("godep", "go", "test", "github.com/google/cadvisor/integration/tests/...", "--host", host, "--port", portStr)
err = RunCommand("godep", "go", "test", "github.com/google/cadvisor/integration/tests/...", "--host", host, "--port", portStr, "--ssh-options", *sshOptions)
if err == nil {
// On success, break out of retry loop
break
@ -172,9 +172,8 @@ func PushAndRunTests(host, testDir string) error {
}
if err != nil {
// Copy logs from the host
args = common.GetGCComputeArgs("copy-files", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
// Declare new error or it will get shadowed by logs, err := <> and we won't be able to unset it from nil
err2 := RunCommand("gcloud", args...)
err2 := RunSshCommand("scp", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
if err2 != nil {
return fmt.Errorf("error fetching logs: %v for %v", err2, err)
}