Register inotify watches on all cgroup hierarchies.

We used to only register them on the first hierarchy that was created (I
think this was unintentional). This caused some weird edgecases where
we'd try to delete a watch event we didn't create. It is an error we
ignore today (since we fix it in < 60s) but delays our destruction of
the container.

container/raw/handler.go

@@ -36,13 +36,24 @@ import (
 )
 
 type rawContainerHandler struct {
+	// Name of the container for this handler.
 	name               string
 	cgroup             *cgroups.Cgroup
 	cgroupSubsystems   *cgroupSubsystems
 	machineInfoFactory info.MachineInfoFactory
+
+	// Inotify event watcher.
 	watcher *inotify.Watcher
+
+	// Signal for watcher thread to stop.
 	stopWatcher chan error
+
+	// Containers being watched for new subcontainers.
 	watches map[string]struct{}
+
+	// Cgroup paths being watchd for new subcontainers
+	cgroupWatches map[string]struct{}
+
 	fsInfo           fs.FsInfo
 	networkInterface *networkInterface
 	externalMounts   []mount
@@ -76,6 +87,7 @@ func newRawContainerHandler(name string, cgroupSubsystems *cgroupSubsystems, mac
 		machineInfoFactory: machineInfoFactory,
 		stopWatcher:        make(chan error),
 		watches:            make(map[string]struct{}),
+		cgroupWatches:      make(map[string]struct{}),
 		fsInfo:             fsInfo,
 		networkInterface:   networkInterface,
 		externalMounts:     externalMounts,
@@ -322,7 +334,9 @@ func (self *rawContainerHandler) watchDirectory(dir string, containerName string
 		return err
 	}
 	self.watches[containerName] = struct{}{}
+	self.cgroupWatches[dir] = struct{}{}
 
+	// TODO(vmarmol): We should re-do this once we're done to ensure directories were not added in the meantime.
 	// Watch subdirectories as well.
 	entries, err := ioutil.ReadDir(dir)
 	if err != nil {
@@ -372,28 +386,33 @@ func (self *rawContainerHandler) processEvent(event *inotify.Event, events chan
 	// Maintain the watch for the new or deleted container.
 	switch {
 	case eventType == container.SubcontainerAdd:
-		// If we've already seen this event, return.
+		_, alreadyWatched := self.watches[containerName]
-		if _, ok := self.watches[containerName]; ok {
-			return nil
-		}
 
 		// New container was created, watch it.
 		err := self.watchDirectory(event.Name, containerName)
 		if err != nil {
 			return err
 		}
-	case eventType == container.SubcontainerDelete:
+
-		// If we've already seen this event, return.
+		// Only report container creation once.
-		if _, ok := self.watches[containerName]; !ok {
+		if alreadyWatched {
 			return nil
 		}
-		delete(self.watches, containerName)
+	case eventType == container.SubcontainerDelete:
-
+		// Container was deleted, stop watching for it. Only delete the event if we registered it.
-		// Container was deleted, stop watching for it.
+		if _, ok := self.cgroupWatches[event.Name]; ok {
 			err := self.watcher.RemoveWatch(event.Name)
 			if err != nil {
 				return err
 			}
+			delete(self.cgroupWatches, event.Name)
+		}
+
+		// Only report container deletion once.
+		if _, ok := self.watches[containerName]; !ok {
+			return nil
+		}
+		delete(self.watches, containerName)
 	default:
 		return fmt.Errorf("unknown event type %v", eventType)
 	}