From a022fa71b7459640ce520ef88d023fdb893998b8 Mon Sep 17 00:00:00 2001 From: Jordan Liggitt Date: Tue, 9 Apr 2019 21:50:26 +0000 Subject: [PATCH] Split docker context initialization --- container/docker/plugin.go | 39 ++++++++++++++++++++++++++++++++++ manager/manager.go | 43 +------------------------------------- 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/container/docker/plugin.go b/container/docker/plugin.go index 63938db8..c4ca43a4 100644 --- a/container/docker/plugin.go +++ b/container/docker/plugin.go @@ -15,12 +15,18 @@ package docker import ( + "time" + "github.com/google/cadvisor/container" "github.com/google/cadvisor/fs" info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/watcher" + "golang.org/x/net/context" + "k8s.io/klog" ) +const dockerClientTimeout = 10 * time.Second + // NewPlugin returns an implementation of container.Plugin suitable for passing to container.RegisterPlugin() func NewPlugin() container.Plugin { return &plugin{} @@ -29,6 +35,14 @@ func NewPlugin() container.Plugin { type plugin struct{} func (p *plugin) InitializeFSContext(context *fs.Context) error { + SetTimeout(dockerClientTimeout) + // Try to connect to docker indefinitely on startup. + dockerStatus := retryDockerStatus() + context.Docker = fs.DockerContext{ + Root: RootDir(), + Driver: dockerStatus.Driver, + DriverStatus: dockerStatus.DriverStatus, + } return nil } @@ -36,3 +50,28 @@ func (p *plugin) Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, inc err := Register(factory, fsInfo, includedMetrics) return nil, err } + +func retryDockerStatus() info.DockerStatus { + startupTimeout := dockerClientTimeout + maxTimeout := 4 * startupTimeout + for { + ctx, _ := context.WithTimeout(context.Background(), startupTimeout) + dockerStatus, err := StatusWithContext(ctx) + if err == nil { + return dockerStatus + } + + switch err { + case context.DeadlineExceeded: + klog.Warningf("Timeout trying to communicate with docker during initialization, will retry") + default: + klog.V(5).Infof("Docker not connected: %v", err) + return info.DockerStatus{} + } + + startupTimeout = 2 * startupTimeout + if startupTimeout > maxTimeout { + startupTimeout = maxTimeout + } + } +} diff --git a/manager/manager.go b/manager/manager.go index e9225aa1..ab1770b8 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -43,7 +43,6 @@ import ( "github.com/google/cadvisor/watcher" "github.com/opencontainers/runc/libcontainer/cgroups" - "golang.org/x/net/context" "k8s.io/klog" "k8s.io/utils/clock" ) @@ -55,8 +54,6 @@ var eventStorageAgeLimit = flag.String("event_storage_age_limit", "default=24h", var eventStorageEventLimit = flag.String("event_storage_event_limit", "default=100000", "Max number of events to store (per type). Value is a comma separated list of key values, where the keys are event types (e.g.: creation, oom) or \"default\" and the value is an integer. Default is applied to all non-specified event types") var applicationMetricsCountLimit = flag.Int("application_metrics_count_limit", 100, "Max number of application metrics to store (per container)") -const dockerClientTimeout = 10 * time.Second - // The Manager interface defines operations for starting a manager and getting // container and machine information. type Manager interface { @@ -148,20 +145,7 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn } klog.V(2).Infof("cAdvisor running in container: %q", selfContainer) - var ( - dockerStatus info.DockerStatus - ) - docker.SetTimeout(dockerClientTimeout) - // Try to connect to docker indefinitely on startup. - dockerStatus = retryDockerStatus() - - context := fs.Context{ - Docker: fs.DockerContext{ - Root: docker.RootDir(), - Driver: dockerStatus.Driver, - DriverStatus: dockerStatus.DriverStatus, - }, - } + context := fs.Context{} if err := container.InitializeFSContext(&context); err != nil { return nil, err @@ -218,31 +202,6 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn return newManager, nil } -func retryDockerStatus() info.DockerStatus { - startupTimeout := dockerClientTimeout - maxTimeout := 4 * startupTimeout - for { - ctx, _ := context.WithTimeout(context.Background(), startupTimeout) - dockerStatus, err := docker.StatusWithContext(ctx) - if err == nil { - return dockerStatus - } - - switch err { - case context.DeadlineExceeded: - klog.Warningf("Timeout trying to communicate with docker during initialization, will retry") - default: - klog.V(5).Infof("Docker not connected: %v", err) - return info.DockerStatus{} - } - - startupTimeout = 2 * startupTimeout - if startupTimeout > maxTimeout { - startupTimeout = maxTimeout - } - } -} - // A namespaced container name. type namespacedContainerName struct { // The namespace of the container. Can be empty for the root namespace.