Split docker context initialization

This commit is contained in:
Jordan Liggitt 2019-04-09 21:50:26 +00:00
parent e9a44a2984
commit a022fa71b7
2 changed files with 40 additions and 42 deletions

View File

@ -15,12 +15,18 @@
package docker
import (
"time"
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/fs"
info "github.com/google/cadvisor/info/v1"
"github.com/google/cadvisor/watcher"
"golang.org/x/net/context"
"k8s.io/klog"
)
const dockerClientTimeout = 10 * time.Second
// NewPlugin returns an implementation of container.Plugin suitable for passing to container.RegisterPlugin()
func NewPlugin() container.Plugin {
return &plugin{}
@ -29,6 +35,14 @@ func NewPlugin() container.Plugin {
type plugin struct{}
func (p *plugin) InitializeFSContext(context *fs.Context) error {
SetTimeout(dockerClientTimeout)
// Try to connect to docker indefinitely on startup.
dockerStatus := retryDockerStatus()
context.Docker = fs.DockerContext{
Root: RootDir(),
Driver: dockerStatus.Driver,
DriverStatus: dockerStatus.DriverStatus,
}
return nil
}
@ -36,3 +50,28 @@ func (p *plugin) Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, inc
err := Register(factory, fsInfo, includedMetrics)
return nil, err
}
func retryDockerStatus() info.DockerStatus {
startupTimeout := dockerClientTimeout
maxTimeout := 4 * startupTimeout
for {
ctx, _ := context.WithTimeout(context.Background(), startupTimeout)
dockerStatus, err := StatusWithContext(ctx)
if err == nil {
return dockerStatus
}
switch err {
case context.DeadlineExceeded:
klog.Warningf("Timeout trying to communicate with docker during initialization, will retry")
default:
klog.V(5).Infof("Docker not connected: %v", err)
return info.DockerStatus{}
}
startupTimeout = 2 * startupTimeout
if startupTimeout > maxTimeout {
startupTimeout = maxTimeout
}
}
}

View File

@ -43,7 +43,6 @@ import (
"github.com/google/cadvisor/watcher"
"github.com/opencontainers/runc/libcontainer/cgroups"
"golang.org/x/net/context"
"k8s.io/klog"
"k8s.io/utils/clock"
)
@ -55,8 +54,6 @@ var eventStorageAgeLimit = flag.String("event_storage_age_limit", "default=24h",
var eventStorageEventLimit = flag.String("event_storage_event_limit", "default=100000", "Max number of events to store (per type). Value is a comma separated list of key values, where the keys are event types (e.g.: creation, oom) or \"default\" and the value is an integer. Default is applied to all non-specified event types")
var applicationMetricsCountLimit = flag.Int("application_metrics_count_limit", 100, "Max number of application metrics to store (per container)")
const dockerClientTimeout = 10 * time.Second
// The Manager interface defines operations for starting a manager and getting
// container and machine information.
type Manager interface {
@ -148,20 +145,7 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn
}
klog.V(2).Infof("cAdvisor running in container: %q", selfContainer)
var (
dockerStatus info.DockerStatus
)
docker.SetTimeout(dockerClientTimeout)
// Try to connect to docker indefinitely on startup.
dockerStatus = retryDockerStatus()
context := fs.Context{
Docker: fs.DockerContext{
Root: docker.RootDir(),
Driver: dockerStatus.Driver,
DriverStatus: dockerStatus.DriverStatus,
},
}
context := fs.Context{}
if err := container.InitializeFSContext(&context); err != nil {
return nil, err
@ -218,31 +202,6 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn
return newManager, nil
}
func retryDockerStatus() info.DockerStatus {
startupTimeout := dockerClientTimeout
maxTimeout := 4 * startupTimeout
for {
ctx, _ := context.WithTimeout(context.Background(), startupTimeout)
dockerStatus, err := docker.StatusWithContext(ctx)
if err == nil {
return dockerStatus
}
switch err {
case context.DeadlineExceeded:
klog.Warningf("Timeout trying to communicate with docker during initialization, will retry")
default:
klog.V(5).Infof("Docker not connected: %v", err)
return info.DockerStatus{}
}
startupTimeout = 2 * startupTimeout
if startupTimeout > maxTimeout {
startupTimeout = maxTimeout
}
}
}
// A namespaced container name.
type namespacedContainerName struct {
// The namespace of the container. Can be empty for the root namespace.