Merge pull request #1327 from pmorie/thinls-binary-check

Check for thin_ls binary in path when using ThinPoolWatcher

container/docker/factory.go

@@ -197,9 +197,11 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, ignoreMetrics c
 	)
 
 	if dockerStorageDriver == devicemapperStorageDriver {
-		// If the storage drive is devicemapper, create and start a
+		_, err := devicemapper.ThinLsBinaryPresent()
-		// ThinPoolWatcher to monitor the size of container CoW layers with
+		if err == nil {
-		// thin_ls.
+			// If the storage driver is devicemapper, create and start a
+			// ThinPoolWatcher to monitor the size of container CoW layers
+			// with thin_ls.
 			dockerThinPoolName, err := dockerutil.DockerThinPoolName(*dockerInfo)
 			if err != nil {
 				return fmt.Errorf("couldn't find device mapper thin pool name: %v", err)
@@ -207,14 +209,26 @@ func Register(factory info.MachineInfoFactory, fsInfo fs.FsInfo, ignoreMetrics c
 
 			dockerMetadataDevice, err := dockerutil.DockerMetadataDevice(*dockerInfo)
 			if err != nil {
-			return fmt.Errorf("couldn't determine devicemapper metadata device")
+				return fmt.Errorf("couldn't determine devicemapper metadata device: %v", err)
+			}
+
+			thinPoolWatcher, err = devicemapper.NewThinPoolWatcher(dockerThinPoolName, dockerMetadataDevice)
+			if err != nil {
+				return fmt.Errorf("couldn't create thin pool watcher: %v", err)
 			}
 
-		thinPoolWatcher = devicemapper.NewThinPoolWatcher(dockerThinPoolName, dockerMetadataDevice)
 			go thinPoolWatcher.Start()
+		} else {
+			msg := []string{
+				"Couldn't locate thin_ls binary; not starting thin pool watcher.",
+				"Containers backed by thin pools will not show accurate usage.",
+				"err: %v",
+			}
+			glog.Errorf(strings.Join(msg, " "), err)
+		}
 	}
 
-	glog.Infof("registering Docker factory")
+	glog.Infof("Registering Docker factory")
 	f := &dockerFactory{
 		cgroupSubsystems:   cgroupSubsystems,
 		client:             client,

devicemapper/dmsetup_client.go

@@ -21,19 +21,26 @@ import (
 	"github.com/golang/glog"
 )
 
-// DmsetupClient is a low-level client for interacting with devicemapper via
+// DmsetupClient is a low-level client for interacting with device mapper via
-// the dmsetup utility.
+// the `dmsetup` utility, which is provided by the `device-mapper` package.
 type DmsetupClient interface {
+	// Table runs `dmsetup table` on the given device name and returns the
+	// output or an error.
 	Table(deviceName string) ([]byte, error)
+	// Message runs `dmsetup message` on the given device, passing the given
+	// message to the given sector, and returns the output or an error.
 	Message(deviceName string, sector int, message string) ([]byte, error)
+	// Status runs `dmsetup status` on the given device and returns the output
+	// or an error.
 	Status(deviceName string) ([]byte, error)
 }
 
+// NewDmSetupClient returns a new DmsetupClient.
 func NewDmsetupClient() DmsetupClient {
 	return &defaultDmsetupClient{}
 }
 
-// defaultDmsetupClient implements the standard behavior for interacting with dmsetup.
+// defaultDmsetupClient is a functional DmsetupClient
 type defaultDmsetupClient struct{}
 
 var _ DmsetupClient = &defaultDmsetupClient{}

devicemapper/fake/dmsetup_client_fake.go

@@ -23,6 +23,7 @@ type DmsetupCommand struct {
 	Err    error
 }
 
+// NewFakeDmsetupClient returns a new fake DmsetupClient.
 func NewFakeDmsetupClient(t *testing.T, commands ...DmsetupCommand) *FakeDmsetupClient {
 	if len(commands) == 0 {
 		commands = make([]DmsetupCommand, 0)
@@ -30,7 +31,8 @@ func NewFakeDmsetupClient(t *testing.T, commands ...DmsetupCommand) *FakeDmsetup
 	return &FakeDmsetupClient{t: t, commands: commands}
 }
 
-// FakeDmsetupClient is a thread-unsafe fake implementation of the DmsetupClient interface
+// FakeDmsetupClient is a thread-unsafe fake implementation of the
+// DmsetupClient interface
 type FakeDmsetupClient struct {
 	t        *testing.T
 	commands []DmsetupCommand

devicemapper/fake/thin_ls_client_fake.go

@@ -18,6 +18,7 @@ type FakeThinLsClient struct {
 	err    error
 }
 
+// NewFakeThinLsClient returns a new fake ThinLsClient.
 func NewFakeThinLsClient(result map[string]uint64, err error) *FakeThinLsClient {
 	return &FakeThinLsClient{result, err}
 }

devicemapper/thin_ls_client.go

@@ -24,24 +24,38 @@ import (
 	"github.com/golang/glog"
 )
 
-// thinLsClient knows how to run a thin_ls very specific to CoW usage for containers.
+// thinLsClient knows how to run a thin_ls very specific to CoW usage for
+// containers.
 type thinLsClient interface {
+	// ThinLs runs a thin ls on the given device, which is expected to be a
+	// metadata device. The caller must hold the metadata snapshot for the
+	// device.
 	ThinLs(deviceName string) (map[string]uint64, error)
 }
 
-func newThinLsClient() thinLsClient {
+// newThinLsClient returns a thinLsClient or an error if the thin_ls binary
-	return &defaultThinLsClient{}
+// couldn't be located.
+func newThinLsClient() (thinLsClient, error) {
+	thinLsPath, err := ThinLsBinaryPresent()
+	if err != nil {
+		return nil, fmt.Errorf("error creating thin_ls client: %v", err)
+	}
+
+	return &defaultThinLsClient{thinLsPath}, nil
 }
 
-type defaultThinLsClient struct{}
+// defaultThinLsClient is a functional thinLsClient
+type defaultThinLsClient struct {
+	thinLsPath string
+}
 
 var _ thinLsClient = &defaultThinLsClient{}
 
-func (*defaultThinLsClient) ThinLs(deviceName string) (map[string]uint64, error) {
+func (c *defaultThinLsClient) ThinLs(deviceName string) (map[string]uint64, error) {
 	args := []string{"--no-headers", "-m", "-o", "DEV,EXCLUSIVE_BYTES", deviceName}
 	glog.V(4).Infof("running command: thin_ls %v", strings.Join(args, " "))
 
-	output, err := exec.Command("thin_ls", args...).Output()
+	output, err := exec.Command(c.thinLsPath, args...).Output()
 	if err != nil {
 		return nil, fmt.Errorf("Error running command `thin_ls %v`: %v\noutput:\n\n%v", strings.Join(args, " "), err, string(output))
 	}
@@ -49,7 +63,8 @@ func (*defaultThinLsClient) ThinLs(deviceName string) (map[string]uint64, error)
 	return parseThinLsOutput(output), nil
 }
 
-// parseThinLsOutput parses the output returned by thin_ls to build a map of device id -> usage.
+// parseThinLsOutput parses the output returned by thin_ls to build a map of
+// device id -> usage.
 func parseThinLsOutput(output []byte) map[string]uint64 {
 	cache := map[string]uint64{}
 

devicemapper/thin_pool_watcher.go

@@ -22,7 +22,8 @@ import (
 	"github.com/golang/glog"
 )
 
-// ThinPoolWatcher maintains a cache of device name -> usage stats for a devicemapper thin-pool using thin_ls.
+// ThinPoolWatcher maintains a cache of device name -> usage stats for a
+// devicemapper thin-pool using thin_ls.
 type ThinPoolWatcher struct {
 	poolName       string
 	metadataDevice string
@@ -34,8 +35,14 @@ type ThinPoolWatcher struct {
 	thinLsClient   thinLsClient
 }
 
-// NewThinPoolWatcher returns a new ThinPoolWatcher for the given devicemapper thin pool name and metadata device.
+// NewThinPoolWatcher returns a new ThinPoolWatcher for the given devicemapper
-func NewThinPoolWatcher(poolName, metadataDevice string) *ThinPoolWatcher {
+// thin pool name and metadata device or an error.
+func NewThinPoolWatcher(poolName, metadataDevice string) (*ThinPoolWatcher, error) {
+	thinLsClient, err := newThinLsClient()
+	if err != nil {
+		return nil, fmt.Errorf("encountered error creating thin_ls client: %v", err)
+	}
+
 	return &ThinPoolWatcher{poolName: poolName,
 		metadataDevice: metadataDevice,
 		lock:           &sync.RWMutex{},
@@ -43,11 +50,11 @@ func NewThinPoolWatcher(poolName, metadataDevice string) *ThinPoolWatcher {
 		period:         15 * time.Second,
 		stopChan:       make(chan struct{}),
 		dmsetup:        NewDmsetupClient(),
-		thinLsClient:   newThinLsClient(),
+		thinLsClient:   thinLsClient,
-	}
+	}, nil
 }
 
-// Start starts the thin pool watcher.
+// Start starts the ThinPoolWatcher.
 func (w *ThinPoolWatcher) Start() {
 	err := w.Refresh()
 	if err != nil {
@@ -72,6 +79,7 @@ func (w *ThinPoolWatcher) Start() {
 	}
 }
 
+// Stop stops the ThinPoolWatcher.
 func (w *ThinPoolWatcher) Stop() {
 	close(w.stopChan)
 }
@@ -80,6 +88,7 @@ func (w *ThinPoolWatcher) Stop() {
 func (w *ThinPoolWatcher) GetUsage(deviceId string) (uint64, error) {
 	w.lock.RLock()
 	defer w.lock.RUnlock()
+
 	v, ok := w.cache[deviceId]
 	if !ok {
 		return 0, fmt.Errorf("no cached value for usage of device %v", deviceId)
@@ -115,7 +124,8 @@ func (w *ThinPoolWatcher) Refresh() error {
 	}
 
 	glog.Infof("reserving metadata snapshot for thin-pool %v", w.poolName)
-	// NOTE: "0" in the call below is for the 'sector' argument to 'dmsetup message'.  It's not needed for thin pools.
+	// NOTE: "0" in the call below is for the 'sector' argument to 'dmsetup
+	// message'.  It's not needed for thin pools.
 	if output, err := w.dmsetup.Message(w.poolName, 0, reserveMetadataMessage); err != nil {
 		err = fmt.Errorf("error reserving metadata for thin-pool %v: %v output: %v", w.poolName, err, string(output))
 		return err
@@ -144,7 +154,8 @@ const (
 	thinPoolDmsetupStatusHeldMetadataRoot = 6
 )
 
-// checkReservation checks to see whether the thin device is currently holding userspace metadata.
+// checkReservation checks to see whether the thin device is currently holding
+// userspace metadata.
 func (w *ThinPoolWatcher) checkReservation(poolName string) (bool, error) {
 	glog.V(5).Infof("checking whether the thin-pool is holding a metadata snapshot")
 	output, err := w.dmsetup.Status(poolName)
@@ -153,7 +164,8 @@ func (w *ThinPoolWatcher) checkReservation(poolName string) (bool, error) {
 	}
 
 	tokens := strings.Split(string(output), " ")
-	// Split returns the input as the last item in the result, adjust the number of tokens by one
+	// Split returns the input as the last item in the result, adjust the
+	// number of tokens by one
 	if len(tokens) != thinPoolDmsetupStatusTokens+1 {
 		return false, fmt.Errorf("unexpected output of dmsetup status command; expected 11 fields, got %v; output: %v", len(tokens), string(output))
 	}

devicemapper/util.go

@@ -0,0 +1,60 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package devicemapper
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// ThinLsBinaryPresent returns the location of the thin_ls binary in the mount
+// namespace cadvisor is running in or an error.  The locations checked are:
+//
+// - /bin/
+// - /usr/sbin/
+// - /usr/bin/
+//
+// ThinLsBinaryPresent checks these paths relative to:
+//
+// 1. For non-containerized operation - `/`
+// 2. For containerized operation - `/rootfs`
+//
+// The thin_ls binary is provided by the device-mapper-persistent-data
+// package.
+func ThinLsBinaryPresent() (string, error) {
+	var (
+		thinLsPath string
+		err        error
+	)
+
+	for _, path := range []string{"/bin", "/usr/sbin/", "/usr/bin"} {
+		// try paths for non-containerized operation
+		// note: thin_ls is most likely a symlink to pdata_tools
+		thinLsPath = filepath.Join(path, "thin_ls")
+		_, err = os.Stat(thinLsPath)
+		if err == nil {
+			return thinLsPath, nil
+		}
+
+		// try paths for containerized operation
+		thinLsPath = filepath.Join("/rootfs", thinLsPath)
+		_, err = os.Stat(thinLsPath)
+		if err == nil {
+			return thinLsPath, nil
+		}
+	}
+
+	return "", fmt.Errorf("unable to find thin_ls binary")
+}