Integration test for Docker and perf stats

Signed-off-by: Maciej "Iwan" Iwanowski <maciej.iwanowski@intel.com>

.github/workflows/test.yml

@@ -41,3 +41,9 @@ jobs:
       - name: Run integration tests
         run: |
           cd $GITHUB_WORKSPACE/go/src/github.com/google/cadvisor && make docker-test-integration
+      - name: Upload cAdvisor log file
+        uses: actions/upload-artifact@v1
+        if: failure()
+        with:
+          name: cadvisor.log
+          path: ${{ github.workspace }}/go/src/github.com/google/cadvisor/cadvisor.log

build/integration-in-docker.sh

@@ -19,35 +19,57 @@ set -e
 ROOT="$(cd "$(dirname "${BASH_SOURCE}")/.." && pwd -P)"
 TMPDIR=$(mktemp -d)
 function delete() {
-    echo "Deleting ${TMPDIR}..."
-    if [[ $EUID -ne 0 ]]; then
-      sudo rm -rf "${TMPDIR}"
-    else
-      rm -rf "${TMPDIR}"
-    fi
+  echo "Deleting ${TMPDIR}..."
+  if [[ $EUID -ne 0 ]]; then
+    sudo rm -rf "${TMPDIR}"
+  else
+    rm -rf "${TMPDIR}"
+  fi
 }
 trap delete EXIT INT
 
-docker run --rm \
-  -w /go/src/github.com/google/cadvisor \
-  -v ${PWD}:/go/src/github.com/google/cadvisor \
-  golang:1.13 \
-  bash -c "env GOOS=linux GO_FLAGS='-race' ./build/build.sh amd64 && \
-    env GOOS=linux go test -c github.com/google/cadvisor/integration/tests/api &&  \
-    env GOOS=linux go test -c github.com/google/cadvisor/integration/tests/healthz"
+function reset() {
+  delete
+  TMPDIR=$(mktemp -d)
+}
 
-EXTRA_DOCKER_OPTS="-e DOCKER_IN_DOCKER_ENABLED=true"
-if [[ "${OSTYPE}" == "linux"* ]]; then
-  EXTRA_DOCKER_OPTS+=" -v ${TMPDIR}/docker-graph:/docker-graph"
-fi
+function run_tests() {
+  GO_FLAGS=${1:-}
+  PACKAGES=${2:-"sudo"}
+  BUILD_PACKAGES=${3:-}
+  CADVISOR_ARGS=${4:-}
+  BUILD_CMD="env GOOS=linux GO_FLAGS='$GO_FLAGS' ./build/build.sh amd64 && \
+    env GOOS=linux GOFLAGS='$GO_FLAGS' go test -c github.com/google/cadvisor/integration/tests/api && \
+    env GOOS=linux GOFLAGS='$GO_FLAGS' go test -c github.com/google/cadvisor/integration/tests/healthz"
 
-mkdir ${TMPDIR}/docker-graph
-docker run --rm \
-  -w /go/src/github.com/google/cadvisor \
-  -v ${ROOT}:/go/src/github.com/google/cadvisor \
-  ${EXTRA_DOCKER_OPTS} \
-  --privileged \
-  --entrypoint="" \
-  gcr.io/k8s-testimages/bootstrap \
-  bash -c "apt update && apt install sudo && \
-/usr/local/bin/runner.sh build/integration.sh"
+  if [ "$BUILD_PACKAGES" != "" ]; then
+    BUILD_CMD="apt-get update && apt-get install $BUILD_PACKAGES && \
+    $BUILD_CMD"
+  fi
+  docker run --rm \
+    -w /go/src/github.com/google/cadvisor \
+    -v ${PWD}:/go/src/github.com/google/cadvisor \
+    golang:1.13 \
+    bash -c "$BUILD_CMD"
+
+  EXTRA_DOCKER_OPTS="-e DOCKER_IN_DOCKER_ENABLED=true"
+  if [[ "${OSTYPE}" == "linux"* ]]; then
+    EXTRA_DOCKER_OPTS+=" -v ${TMPDIR}/docker-graph:/docker-graph"
+  fi
+
+  mkdir ${TMPDIR}/docker-graph
+  docker run --rm \
+    -w /go/src/github.com/google/cadvisor \
+    -v ${ROOT}:/go/src/github.com/google/cadvisor \
+    ${EXTRA_DOCKER_OPTS} \
+    --privileged \
+    --cap-add="sys_admin" \
+    --entrypoint="" \
+    gcr.io/k8s-testimages/bootstrap \
+    bash -c "apt update && apt install $PACKAGES && \
+CADVISOR_ARGS="$CADVISOR_ARGS" /usr/local/bin/runner.sh build/integration.sh"
+}
+
+run_tests "-race" "sudo"
+reset
+run_tests "-tags=libpfm,netgo -race" "sudo libpfm4" "libpfm4 libpfm4-dev" "-perf_events_config=perf/testing/perf-non-hardware.json"

build/integration.sh

@@ -26,7 +26,7 @@ printf "" # Refresh sudo credentials if necessary.
 function start {
   set +e  # We want to handle errors if cAdvisor crashes.
   echo ">> starting cAdvisor locally"
-  GORACE="halt_on_error=1" ./cadvisor --docker_env_metadata_whitelist=TEST_VAR --v=4 --logtostderr &> "$log_file"
+  GORACE="halt_on_error=1" ./cadvisor --docker_env_metadata_whitelist=TEST_VAR --v=6 --logtostderr $CADVISOR_ARGS &>> "$log_file"
   if [ $? != 0 ]; then
     echo "!! cAdvisor exited unexpectedly with Exit $?"
     kill $TEST_PID # cAdvisor crashed: abort testing.
@@ -56,5 +56,5 @@ while [ "$(curl -Gs http://localhost:8080/healthz)" != "ok" ]; do
 done
 
 echo ">> running integration tests against local cAdvisor"
-./api.test --vmodule=*=2
-./healthz.test --vmodule=*=2
+./api.test --vmodule=*=2 -test.v
+./healthz.test --vmodule=*=2 -test.v

integration/tests/api/perf_test.go

@@ -0,0 +1,57 @@
+// +build libpfm,cgo
+
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"testing"
+	"time"
+
+	v1 "github.com/google/cadvisor/info/v1"
+	"github.com/google/cadvisor/integration/framework"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPerfEvents(t *testing.T) {
+	fm := framework.New(t)
+	defer fm.Cleanup()
+
+	containerID := fm.Docker().RunPause()
+	waitForContainerInfo(fm, containerID)
+
+	info, err := fm.Cadvisor().Client().DockerContainer(containerID, &v1.ContainerInfoRequest{
+		NumStats: 1,
+	})
+
+	assert.Nil(t, err)
+	assert.Len(t, info.Stats, 1)
+	assert.Greater(t, len(info.Stats[0].PerfStats), 0, "Length of info.Stats[0].PerfStats is not greater tahn zero")
+	for k, stat := range info.Stats[0].PerfStats {
+		//Everything beyond name is non-deterministic unfortunately.
+		assert.Contains(t, []string{"context-switches", "cpu-migrations-custom"}, stat.Name, "Wrong metric name for key %d: %#v", k, stat)
+	}
+}
+
+func waitForContainerInfo(fm framework.Framework, containerID string) {
+	framework.RetryForDuration(func() error {
+		_, err := fm.Cadvisor().Client().DockerContainer(containerID, &v1.ContainerInfoRequest{NumStats: 1})
+		if err != nil {
+			return err
+		}
+		return nil
+	}, 5*time.Second)
+}

perf/testing/perf-non-hardware.json

@@ -0,0 +1,15 @@
+{
+  "events": [
+    ["context-switches"],
+    ["cpu-migrations-custom"]
+  ],
+  "custom_events": [
+    {
+      "type": 1,
+      "config": [
+        "0x4"
+      ],
+      "name": "cpu-migrations-custom"
+    }
+  ]
+}