From 40e6acb3bb9539f88a644c2c5e5f3c23d076b0f0 Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Fri, 29 Mar 2019 10:14:24 -0400 Subject: [PATCH] Reorganize code to allow conditional enablement of runtimes Change-Id: I76583736d7ad39190a1a2bca820d4e957caadc84 --- cadvisor.go | 2 +- cadvisor_helper.go | 167 ++++++++++++++ container/containerd/factory.go | 2 +- container/containerd/init.go | 33 +++ container/crio/factory.go | 2 +- container/crio/init.go | 33 +++ container/docker/factory.go | 2 +- container/docker/init.go | 33 +++ container/factory.go | 38 ++- container/factory_test.go | 2 +- container/mesos/factory.go | 2 +- container/mesos/init.go | 33 +++ container/raw/factory.go | 2 +- .../raw/raw.go => container/raw/watcher.go | 2 +- container/rkt/factory.go | 2 +- container/rkt/init.go | 36 +++ .../rkt/rkt.go => container/rkt/watcher.go | 5 +- container/systemd/factory.go | 2 +- container/systemd/init.go | 33 +++ manager/manager.go | 218 ++++-------------- manager/manager_test.go | 9 - {manager/watcher => watcher}/watcher.go | 0 22 files changed, 462 insertions(+), 196 deletions(-) create mode 100644 cadvisor_helper.go create mode 100644 container/containerd/init.go create mode 100644 container/crio/init.go create mode 100644 container/docker/init.go create mode 100644 container/mesos/init.go rename manager/watcher/raw/raw.go => container/raw/watcher.go (99%) create mode 100644 container/rkt/init.go rename manager/watcher/rkt/rkt.go => container/rkt/watcher.go (97%) create mode 100644 container/systemd/init.go rename {manager/watcher => watcher}/watcher.go (100%) diff --git a/cadvisor.go b/cadvisor.go index 02b1ea69..13394546 100644 --- a/cadvisor.go +++ b/cadvisor.go @@ -150,7 +150,7 @@ func main() { collectorHttpClient := createCollectorHttpClient(*collectorCert, *collectorKey) - containerManager, err := manager.New(memoryStorage, sysFs, *maxHousekeepingInterval, *allowDynamicHousekeeping, includedMetrics, &collectorHttpClient, strings.Split(*rawCgroupPrefixWhiteList, ",")) + containerManager, err := New(memoryStorage, sysFs, *maxHousekeepingInterval, *allowDynamicHousekeeping, includedMetrics, &collectorHttpClient, strings.Split(*rawCgroupPrefixWhiteList, ",")) if err != nil { klog.Fatalf("Failed to create a Container Manager: %s", err) } diff --git a/cadvisor_helper.go b/cadvisor_helper.go new file mode 100644 index 00000000..eb24004d --- /dev/null +++ b/cadvisor_helper.go @@ -0,0 +1,167 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "net/http" + "os" + "time" + + "github.com/google/cadvisor/accelerators" + "github.com/google/cadvisor/cache/memory" + "github.com/google/cadvisor/container" + _ "github.com/google/cadvisor/container/containerd" + "github.com/google/cadvisor/container/crio" + "github.com/google/cadvisor/container/docker" + _ "github.com/google/cadvisor/container/mesos" + _ "github.com/google/cadvisor/container/raw" + "github.com/google/cadvisor/container/rkt" + _ "github.com/google/cadvisor/container/systemd" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/machine" + "github.com/google/cadvisor/manager" + "github.com/google/cadvisor/utils/sysfs" + "github.com/google/cadvisor/watcher" + + "github.com/opencontainers/runc/libcontainer/cgroups" + "k8s.io/klog" +) + +const dockerClientTimeout = 10 * time.Second + +// New takes a memory storage and returns a new manager. +func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingInterval time.Duration, allowDynamicHousekeeping bool, includedMetricsSet container.MetricSet, collectorHttpClient *http.Client, rawContainerCgroupPathPrefixWhiteList []string) (manager.Manager, error) { + if memoryCache == nil { + return nil, fmt.Errorf("manager requires memory storage") + } + + // Detect the container we are running on. + selfContainer, err := cgroups.GetOwnCgroupPath("cpu") + if err != nil { + return nil, err + } + klog.V(2).Infof("cAdvisor running in container: %q", selfContainer) + + var ( + dockerStatus info.DockerStatus + rktPath string + ) + docker.SetTimeout(dockerClientTimeout) + // Try to connect to docker indefinitely on startup. + dockerStatus = retryDockerStatus() + + if tmpRktPath, err := rkt.RktPath(); err != nil { + klog.V(5).Infof("Rkt not connected: %v", err) + } else { + rktPath = tmpRktPath + } + + crioClient, err := crio.Client() + if err != nil { + return nil, err + } + crioInfo, err := crioClient.Info() + if err != nil { + klog.V(5).Infof("CRI-O not connected: %v", err) + } + + context := fs.Context{ + Docker: fs.DockerContext{ + Root: docker.RootDir(), + Driver: dockerStatus.Driver, + DriverStatus: dockerStatus.DriverStatus, + }, + RktPath: rktPath, + Crio: fs.CrioContext{ + Root: crioInfo.StorageRoot, + }, + } + fsInfo, err := fs.NewFsInfo(context) + if err != nil { + return nil, err + } + + // If cAdvisor was started with host's rootfs mounted, assume that its running + // in its own namespaces. + inHostNamespace := false + if _, err := os.Stat("/rootfs/proc"); os.IsNotExist(err) { + inHostNamespace = true + } + + // Register for new subcontainers. + eventsChannel := make(chan watcher.ContainerEvent, 16) + + machineInfo, err := machine.Info(sysfs, fsInfo, inHostNamespace) + if err != nil { + return nil, err + } + klog.V(1).Infof("Machine: %+v", *machineInfo) + + newManager := manager.New( + memoryCache, + fsInfo, + sysfs, + *machineInfo, + make([]chan error, 0, 2), + selfContainer, + inHostNamespace, + time.Now(), + maxHousekeepingInterval, + allowDynamicHousekeeping, + includedMetricsSet, + []watcher.ContainerWatcher{}, + eventsChannel, + collectorHttpClient, + &accelerators.NvidiaManager{}, + rawContainerCgroupPathPrefixWhiteList) + + versionInfo, err := newManager.GetVersionInfo() + if err != nil { + return nil, err + } + klog.V(1).Infof("Version: %+v", *versionInfo) + return newManager, nil +} + +func retryDockerStatus() info.DockerStatus { + startupTimeout := dockerClientTimeout + maxTimeout := 4 * startupTimeout + for { + ctx, e := context.WithTimeout(context.Background(), startupTimeout) + if e != nil { + klog.V(5).Infof("error during timeout: %v", e) + } + dockerStatus, err := docker.StatusWithContext(ctx) + if err == nil { + return dockerStatus + } + + switch err { + case context.DeadlineExceeded: + klog.Warningf("Timeout trying to communicate with docker during initialization, will retry") + default: + klog.V(5).Infof("Docker not connected: %v", err) + return info.DockerStatus{} + } + + startupTimeout = 2 * startupTimeout + if startupTimeout > maxTimeout { + startupTimeout = maxTimeout + } + } +} diff --git a/container/containerd/factory.go b/container/containerd/factory.go index 87cf0e99..f80f0a97 100644 --- a/container/containerd/factory.go +++ b/container/containerd/factory.go @@ -28,7 +28,7 @@ import ( "github.com/google/cadvisor/container/libcontainer" "github.com/google/cadvisor/fs" info "github.com/google/cadvisor/info/v1" - "github.com/google/cadvisor/manager/watcher" + "github.com/google/cadvisor/watcher" ) var ArgContainerdEndpoint = flag.String("containerd", "/run/containerd/containerd.sock", "containerd endpoint") diff --git a/container/containerd/init.go b/container/containerd/init.go new file mode 100644 index 00000000..2509e19d --- /dev/null +++ b/container/containerd/init.go @@ -0,0 +1,33 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package containerd + +import ( + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/watcher" + "k8s.io/klog" +) + +func init() { + err := container.RegisterPlugin("containerd", func(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics container.MetricSet) (watcher.ContainerWatcher, error) { + err := Register(factory, fsInfo, includedMetrics) + return nil, err + }) + if err != nil { + klog.Fatalf("Failed to register containerd plugin: %v", err) + } +} diff --git a/container/crio/factory.go b/container/crio/factory.go index b3380b02..b530ee4a 100644 --- a/container/crio/factory.go +++ b/container/crio/factory.go @@ -24,7 +24,7 @@ import ( "github.com/google/cadvisor/container/libcontainer" "github.com/google/cadvisor/fs" info "github.com/google/cadvisor/info/v1" - "github.com/google/cadvisor/manager/watcher" + "github.com/google/cadvisor/watcher" "k8s.io/klog" ) diff --git a/container/crio/init.go b/container/crio/init.go new file mode 100644 index 00000000..5437b3eb --- /dev/null +++ b/container/crio/init.go @@ -0,0 +1,33 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package crio + +import ( + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/watcher" + "k8s.io/klog" +) + +func init() { + err := container.RegisterPlugin("crio", func(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics container.MetricSet) (watcher.ContainerWatcher, error) { + err := Register(factory, fsInfo, includedMetrics) + return nil, err + }) + if err != nil { + klog.Fatalf("Failed to register crio plugin: %v", err) + } +} diff --git a/container/docker/factory.go b/container/docker/factory.go index 5802be03..bc51512a 100644 --- a/container/docker/factory.go +++ b/container/docker/factory.go @@ -31,8 +31,8 @@ import ( "github.com/google/cadvisor/fs" info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/machine" - "github.com/google/cadvisor/manager/watcher" dockerutil "github.com/google/cadvisor/utils/docker" + "github.com/google/cadvisor/watcher" "github.com/google/cadvisor/zfs" docker "github.com/docker/docker/client" diff --git a/container/docker/init.go b/container/docker/init.go new file mode 100644 index 00000000..0f5fe39f --- /dev/null +++ b/container/docker/init.go @@ -0,0 +1,33 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import ( + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/watcher" + "k8s.io/klog" +) + +func init() { + err := container.RegisterPlugin("docker", func(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics container.MetricSet) (watcher.ContainerWatcher, error) { + err := Register(factory, fsInfo, includedMetrics) + return nil, err + }) + if err != nil { + klog.Fatalf("Failed to register docker plugin: %v", err) + } +} diff --git a/container/factory.go b/container/factory.go index ae03960e..900af747 100644 --- a/container/factory.go +++ b/container/factory.go @@ -18,7 +18,9 @@ import ( "fmt" "sync" - "github.com/google/cadvisor/manager/watcher" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/watcher" "k8s.io/klog" ) @@ -71,6 +73,40 @@ func (ms MetricSet) Add(mk MetricKind) { ms[mk] = struct{}{} } +type Register func(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics MetricSet) (watcher.ContainerWatcher, error) + +// All registered auth provider plugins. +var pluginsLock sync.Mutex +var plugins = make(map[string]Register) + +func RegisterPlugin(name string, plugin Register) error { + pluginsLock.Lock() + defer pluginsLock.Unlock() + if _, found := plugins[name]; found { + return fmt.Errorf("Plugin %q was registered twice", name) + } + klog.V(4).Infof("Registered Plugin %q", name) + plugins[name] = plugin + return nil +} + +func InitializePlugins(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics MetricSet) []watcher.ContainerWatcher { + pluginsLock.Lock() + defer pluginsLock.Unlock() + + containerWatchers := []watcher.ContainerWatcher{} + for name, register := range plugins { + watcher, err := register(factory, fsInfo, includedMetrics) + if err != nil { + klog.V(5).Infof("Registration of the %s container factory failed: %v", name, err) + } + if watcher != nil { + containerWatchers = append(containerWatchers, watcher) + } + } + return containerWatchers +} + // TODO(vmarmol): Consider not making this global. // Global list of factories. var ( diff --git a/container/factory_test.go b/container/factory_test.go index 627ef628..b956db06 100644 --- a/container/factory_test.go +++ b/container/factory_test.go @@ -19,7 +19,7 @@ import ( "github.com/google/cadvisor/container" containertest "github.com/google/cadvisor/container/testing" - "github.com/google/cadvisor/manager/watcher" + "github.com/google/cadvisor/watcher" "github.com/stretchr/testify/mock" ) diff --git a/container/mesos/factory.go b/container/mesos/factory.go index dd610a43..c9b55c9b 100644 --- a/container/mesos/factory.go +++ b/container/mesos/factory.go @@ -26,7 +26,7 @@ import ( "github.com/google/cadvisor/container/libcontainer" "github.com/google/cadvisor/fs" info "github.com/google/cadvisor/info/v1" - "github.com/google/cadvisor/manager/watcher" + "github.com/google/cadvisor/watcher" "k8s.io/klog" ) diff --git a/container/mesos/init.go b/container/mesos/init.go new file mode 100644 index 00000000..cccde2ce --- /dev/null +++ b/container/mesos/init.go @@ -0,0 +1,33 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mesos + +import ( + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/watcher" + "k8s.io/klog" +) + +func init() { + err := container.RegisterPlugin("mesos", func(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics container.MetricSet) (watcher.ContainerWatcher, error) { + err := Register(factory, fsInfo, includedMetrics) + return nil, err + }) + if err != nil { + klog.Fatalf("Failed to register mesos plugin: %v", err) + } +} diff --git a/container/raw/factory.go b/container/raw/factory.go index c42e473f..cd913d28 100644 --- a/container/raw/factory.go +++ b/container/raw/factory.go @@ -24,7 +24,7 @@ import ( "github.com/google/cadvisor/container/libcontainer" "github.com/google/cadvisor/fs" info "github.com/google/cadvisor/info/v1" - watch "github.com/google/cadvisor/manager/watcher" + watch "github.com/google/cadvisor/watcher" "k8s.io/klog" ) diff --git a/manager/watcher/raw/raw.go b/container/raw/watcher.go similarity index 99% rename from manager/watcher/raw/raw.go rename to container/raw/watcher.go index 9dee46b6..201b870b 100644 --- a/manager/watcher/raw/raw.go +++ b/container/raw/watcher.go @@ -25,7 +25,7 @@ import ( "github.com/google/cadvisor/container/common" "github.com/google/cadvisor/container/libcontainer" - "github.com/google/cadvisor/manager/watcher" + "github.com/google/cadvisor/watcher" inotify "github.com/sigma/go-inotify" "k8s.io/klog" diff --git a/container/rkt/factory.go b/container/rkt/factory.go index ab339708..859eee38 100644 --- a/container/rkt/factory.go +++ b/container/rkt/factory.go @@ -21,7 +21,7 @@ import ( "github.com/google/cadvisor/container/libcontainer" "github.com/google/cadvisor/fs" info "github.com/google/cadvisor/info/v1" - "github.com/google/cadvisor/manager/watcher" + "github.com/google/cadvisor/watcher" "k8s.io/klog" ) diff --git a/container/rkt/init.go b/container/rkt/init.go new file mode 100644 index 00000000..dc104e07 --- /dev/null +++ b/container/rkt/init.go @@ -0,0 +1,36 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rkt + +import ( + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/watcher" + "k8s.io/klog" +) + +func init() { + err := container.RegisterPlugin("rkt", func(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics container.MetricSet) (watcher.ContainerWatcher, error) { + err := Register(factory, fsInfo, includedMetrics) + if err != nil { + return nil, err + } + return NewRktContainerWatcher() + }) + if err != nil { + klog.Fatalf("Failed to register rkt plugin: %v", err) + } +} diff --git a/manager/watcher/rkt/rkt.go b/container/rkt/watcher.go similarity index 97% rename from manager/watcher/rkt/rkt.go rename to container/rkt/watcher.go index a2d910f8..d7395cab 100644 --- a/manager/watcher/rkt/rkt.go +++ b/container/rkt/watcher.go @@ -19,8 +19,7 @@ import ( "path/filepath" "time" - "github.com/google/cadvisor/container/rkt" - "github.com/google/cadvisor/manager/watcher" + "github.com/google/cadvisor/watcher" rktapi "github.com/coreos/rkt/api/v1alpha" "golang.org/x/net/context" @@ -118,7 +117,7 @@ func (self *rktContainerWatcher) sendDestroyEvent(cgroup string, events chan wat } func listRunningPods() ([]*rktapi.Pod, error) { - client, err := rkt.Client() + client, err := Client() if err != nil { return nil, err } diff --git a/container/systemd/factory.go b/container/systemd/factory.go index 100c79e1..8538fb40 100644 --- a/container/systemd/factory.go +++ b/container/systemd/factory.go @@ -21,7 +21,7 @@ import ( "github.com/google/cadvisor/container" "github.com/google/cadvisor/fs" info "github.com/google/cadvisor/info/v1" - "github.com/google/cadvisor/manager/watcher" + "github.com/google/cadvisor/watcher" "k8s.io/klog" ) diff --git a/container/systemd/init.go b/container/systemd/init.go new file mode 100644 index 00000000..1c91023d --- /dev/null +++ b/container/systemd/init.go @@ -0,0 +1,33 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package systemd + +import ( + "github.com/google/cadvisor/container" + "github.com/google/cadvisor/fs" + info "github.com/google/cadvisor/info/v1" + "github.com/google/cadvisor/watcher" + "k8s.io/klog" +) + +func init() { + err := container.RegisterPlugin("systemd", func(factory info.MachineInfoFactory, fsInfo fs.FsInfo, includedMetrics container.MetricSet) (watcher.ContainerWatcher, error) { + err := Register(factory, fsInfo, includedMetrics) + return nil, err + }) + if err != nil { + klog.Fatalf("Failed to register systemd plugin: %v", err) + } +} diff --git a/manager/manager.go b/manager/manager.go index 2fae1190..5f4fc251 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -19,7 +19,6 @@ import ( "flag" "fmt" "net/http" - "os" "path" "strconv" "strings" @@ -30,27 +29,18 @@ import ( "github.com/google/cadvisor/cache/memory" "github.com/google/cadvisor/collector" "github.com/google/cadvisor/container" - "github.com/google/cadvisor/container/containerd" - "github.com/google/cadvisor/container/crio" "github.com/google/cadvisor/container/docker" - "github.com/google/cadvisor/container/mesos" "github.com/google/cadvisor/container/raw" - "github.com/google/cadvisor/container/rkt" - "github.com/google/cadvisor/container/systemd" "github.com/google/cadvisor/events" "github.com/google/cadvisor/fs" info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/info/v2" "github.com/google/cadvisor/machine" - "github.com/google/cadvisor/manager/watcher" - rawwatcher "github.com/google/cadvisor/manager/watcher/raw" - rktwatcher "github.com/google/cadvisor/manager/watcher/rkt" "github.com/google/cadvisor/utils/oomparser" "github.com/google/cadvisor/utils/sysfs" "github.com/google/cadvisor/version" + "github.com/google/cadvisor/watcher" - "github.com/opencontainers/runc/libcontainer/cgroups" - "golang.org/x/net/context" "k8s.io/klog" "k8s.io/utils/clock" ) @@ -62,8 +52,6 @@ var eventStorageAgeLimit = flag.String("event_storage_age_limit", "default=24h", var eventStorageEventLimit = flag.String("event_storage_event_limit", "default=100000", "Max number of events to store (per type). Value is a comma separated list of key values, where the keys are event types (e.g.: creation, oom) or \"default\" and the value is an integer. Default is applied to all non-specified event types") var applicationMetricsCountLimit = flag.Int("application_metrics_count_limit", 100, "Max number of application metrics to store (per container)") -const dockerClientTimeout = 10 * time.Second - // The Manager interface defines operations for starting a manager and getting // container and machine information. type Manager interface { @@ -142,129 +130,6 @@ type Manager interface { DebugInfo() map[string][]string } -// New takes a memory storage and returns a new manager. -func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingInterval time.Duration, allowDynamicHousekeeping bool, includedMetricsSet container.MetricSet, collectorHttpClient *http.Client, rawContainerCgroupPathPrefixWhiteList []string) (Manager, error) { - if memoryCache == nil { - return nil, fmt.Errorf("manager requires memory storage") - } - - // Detect the container we are running on. - selfContainer, err := cgroups.GetOwnCgroupPath("cpu") - if err != nil { - return nil, err - } - klog.V(2).Infof("cAdvisor running in container: %q", selfContainer) - - var ( - dockerStatus info.DockerStatus - rktPath string - ) - docker.SetTimeout(dockerClientTimeout) - // Try to connect to docker indefinitely on startup. - dockerStatus = retryDockerStatus() - - if tmpRktPath, err := rkt.RktPath(); err != nil { - klog.V(5).Infof("Rkt not connected: %v", err) - } else { - rktPath = tmpRktPath - } - - crioClient, err := crio.Client() - if err != nil { - return nil, err - } - crioInfo, err := crioClient.Info() - if err != nil { - klog.V(5).Infof("CRI-O not connected: %v", err) - } - - context := fs.Context{ - Docker: fs.DockerContext{ - Root: docker.RootDir(), - Driver: dockerStatus.Driver, - DriverStatus: dockerStatus.DriverStatus, - }, - RktPath: rktPath, - Crio: fs.CrioContext{ - Root: crioInfo.StorageRoot, - }, - } - fsInfo, err := fs.NewFsInfo(context) - if err != nil { - return nil, err - } - - // If cAdvisor was started with host's rootfs mounted, assume that its running - // in its own namespaces. - inHostNamespace := false - if _, err := os.Stat("/rootfs/proc"); os.IsNotExist(err) { - inHostNamespace = true - } - - // Register for new subcontainers. - eventsChannel := make(chan watcher.ContainerEvent, 16) - - newManager := &manager{ - containers: make(map[namespacedContainerName]*containerData), - quitChannels: make([]chan error, 0, 2), - memoryCache: memoryCache, - fsInfo: fsInfo, - sysFs: sysfs, - cadvisorContainer: selfContainer, - inHostNamespace: inHostNamespace, - startupTime: time.Now(), - maxHousekeepingInterval: maxHousekeepingInterval, - allowDynamicHousekeeping: allowDynamicHousekeeping, - includedMetrics: includedMetricsSet, - containerWatchers: []watcher.ContainerWatcher{}, - eventsChannel: eventsChannel, - collectorHttpClient: collectorHttpClient, - nvidiaManager: &accelerators.NvidiaManager{}, - rawContainerCgroupPathPrefixWhiteList: rawContainerCgroupPathPrefixWhiteList, - } - - machineInfo, err := machine.Info(sysfs, fsInfo, inHostNamespace) - if err != nil { - return nil, err - } - newManager.machineInfo = *machineInfo - klog.V(1).Infof("Machine: %+v", newManager.machineInfo) - - versionInfo, err := getVersionInfo() - if err != nil { - return nil, err - } - klog.V(1).Infof("Version: %+v", *versionInfo) - - newManager.eventHandler = events.NewEventManager(parseEventsStoragePolicy()) - return newManager, nil -} - -func retryDockerStatus() info.DockerStatus { - startupTimeout := dockerClientTimeout - maxTimeout := 4 * startupTimeout - for { - ctx, _ := context.WithTimeout(context.Background(), startupTimeout) - dockerStatus, err := docker.StatusWithContext(ctx) - if err == nil { - return dockerStatus - } - - switch err { - case context.DeadlineExceeded: - klog.Warningf("Timeout trying to communicate with docker during initialization, will retry") - default: - klog.V(5).Infof("Docker not connected: %v", err) - return info.DockerStatus{} - } - - startupTimeout = 2 * startupTimeout - if startupTimeout > maxTimeout { - startupTimeout = maxTimeout - } - } -} - // A namespaced container name. type namespacedContainerName struct { // The namespace of the container. Can be empty for the root namespace. @@ -274,6 +139,47 @@ type namespacedContainerName struct { Name string } +func New( + memoryCache *memory.InMemoryCache, + fsInfo fs.FsInfo, + sysFs sysfs.SysFs, + machineInfo info.MachineInfo, + quitChannels []chan error, + cadvisorContainer string, + inHostNamespace bool, + startupTime time.Time, + maxHousekeepingInterval time.Duration, + allowDynamicHousekeeping bool, + includedMetrics container.MetricSet, + containerWatchers []watcher.ContainerWatcher, + eventsChannel chan watcher.ContainerEvent, + collectorHttpClient *http.Client, + nvidiaManager accelerators.AcceleratorManager, + rawContainerCgroupPathPrefixWhiteList []string, +) Manager { + impl := &manager{ + containers: make(map[namespacedContainerName]*containerData), + memoryCache: memoryCache, + fsInfo: fsInfo, + sysFs: sysFs, + machineInfo: machineInfo, + quitChannels: quitChannels, + cadvisorContainer: cadvisorContainer, + inHostNamespace: inHostNamespace, + startupTime: startupTime, + maxHousekeepingInterval: maxHousekeepingInterval, + allowDynamicHousekeeping: allowDynamicHousekeeping, + includedMetrics: includedMetrics, + containerWatchers: containerWatchers, + eventsChannel: eventsChannel, + collectorHttpClient: collectorHttpClient, + nvidiaManager: nvidiaManager, + rawContainerCgroupPathPrefixWhiteList: rawContainerCgroupPathPrefixWhiteList, + } + impl.eventHandler = events.NewEventManager(parseEventsStoragePolicy()) + return impl +} + type manager struct { containers map[namespacedContainerName]*containerData containersLock sync.RWMutex @@ -300,48 +206,14 @@ type manager struct { // Start the container manager. func (self *manager) Start() error { - err := docker.Register(self, self.fsInfo, self.includedMetrics) - if err != nil { - klog.V(5).Infof("Registration of the Docker container factory failed: %v.", err) - } + self.containerWatchers = container.InitializePlugins(self, self.fsInfo, self.includedMetrics) - err = rkt.Register(self, self.fsInfo, self.includedMetrics) - if err != nil { - klog.V(5).Infof("Registration of the rkt container factory failed: %v", err) - } else { - watcher, err := rktwatcher.NewRktContainerWatcher() - if err != nil { - return err - } - self.containerWatchers = append(self.containerWatchers, watcher) - } - - err = containerd.Register(self, self.fsInfo, self.includedMetrics) - if err != nil { - klog.V(5).Infof("Registration of the containerd container factory failed: %v", err) - } - - err = crio.Register(self, self.fsInfo, self.includedMetrics) - if err != nil { - klog.V(5).Infof("Registration of the crio container factory failed: %v", err) - } - - err = mesos.Register(self, self.fsInfo, self.includedMetrics) - if err != nil { - klog.V(5).Infof("Registration of the mesos container factory failed: %v", err) - } - - err = systemd.Register(self, self.fsInfo, self.includedMetrics) - if err != nil { - klog.V(5).Infof("Registration of the systemd container factory failed: %v", err) - } - - err = raw.Register(self, self.fsInfo, self.includedMetrics, self.rawContainerCgroupPathPrefixWhiteList) + err := raw.Register(self, self.fsInfo, self.includedMetrics, self.rawContainerCgroupPathPrefixWhiteList) if err != nil { klog.Errorf("Registration of the raw container factory failed: %v", err) } - rawWatcher, err := rawwatcher.NewRawContainerWatcher() + rawWatcher, err := raw.NewRawContainerWatcher() if err != nil { return err } diff --git a/manager/manager_test.go b/manager/manager_test.go index d32f5634..07c1debe 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -23,8 +23,6 @@ import ( "testing" "time" - "net/http" - "github.com/google/cadvisor/cache/memory" "github.com/google/cadvisor/collector" "github.com/google/cadvisor/container" @@ -365,10 +363,3 @@ func TestDockerContainersInfo(t *testing.T) { t.Errorf("expected error %q but received %q", expectedError, err) } } - -func TestNewNilManager(t *testing.T) { - _, err := New(nil, nil, 60*time.Second, true, container.MetricSet{}, http.DefaultClient, []string{"/"}) - if err == nil { - t.Fatalf("Expected nil manager to return error") - } -} diff --git a/manager/watcher/watcher.go b/watcher/watcher.go similarity index 100% rename from manager/watcher/watcher.go rename to watcher/watcher.go