bump(github.com/opencontainers/runc/libcontainer)

Fixes issues with breaking changes to ``GetPids` which is affecting
downstream consumers of cadvisor (e.g. Kubernetes).

Godeps/Godeps.json

@@ -123,8 +123,8 @@
 		},
 		{
 			"ImportPath": "github.com/opencontainers/runc/libcontainer",
-			"Comment": "v0.0.5",
+			"Comment": "v0.0.7",
-			"Rev": "97bc9a7faf3dd660d9be90a2880b2e37f3cdbf38"
+			"Rev": "3268a1ea007f54f16e4d969c0900efb59beb3d08"
 		},
 		{
 			"ImportPath": "github.com/pborman/uuid",

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/README.md

@@ -10,18 +10,34 @@ host system and which is (optionally) isolated from other containers in the syst
 
 #### Using libcontainer
 
-To create a container you first have to initialize an instance of a factory
+Because containers are spawned in a two step process you will need a binary that
-that will handle the creation and initialization for a container.
+will be executed as the init process for the container. In libcontainer, we use
-
+the current binary (/proc/self/exe) to be executed as the init process, and use
-Because containers are spawned in a two step process you will need to provide
+arg "init", we call the first step process "bootstrap", so you always need a "init"
-arguments to a binary that will be executed as the init process for the container.
+function as the entry of "bootstrap".
-To use the current binary that is spawning the containers and acting as the parent
-you can use `os.Args[0]` and we have a command called `init` setup.
 
 ```go
-root, err := libcontainer.New("/var/lib/container", libcontainer.InitArgs(os.Args[0], "init"))
+func init() {
+	if len(os.Args) > 1 && os.Args[1] == "init" {
+		runtime.GOMAXPROCS(1)
+		runtime.LockOSThread()
+		factory, _ := libcontainer.New("")
+		if err := factory.StartInitialization(); err != nil {
+			logrus.Fatal(err)
+		}
+		panic("--this line should have never been executed, congratulations--")
+	}
+}
+```
+
+Then to create a container you first have to initialize an instance of a factory
+that will handle the creation and initialization for a container.
+
+```go
+factory, err := libcontainer.New("/var/lib/container", libcontainer.Cgroupfs, libcontainer.InitArgs(os.Args[0], "init"))
 if err != nil {
-    log.Fatal(err)
+	logrus.Fatal(err)
+	return
 }
 ```
 
@@ -29,8 +45,9 @@ Once you have an instance of the factory created we can create a configuration
 struct describing how the container is to be created. A sample would look similar to this:
 
 ```go
+defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV
 config := &configs.Config{
-    Rootfs: rootfs,
+	Rootfs: "/your/path/to/rootfs",
 	Capabilities: []string{
 		"CAP_CHOWN",
 		"CAP_DAC_OVERRIDE",
@@ -57,12 +74,61 @@ config := &configs.Config{
 	Cgroups: &configs.Cgroup{
 		Name:   "test-container",
 		Parent: "system",
+		Resources: &configs.Resources{
+			MemorySwappiness: -1,
 			AllowAllDevices:  false,
 			AllowedDevices:   configs.DefaultAllowedDevices,
 		},
-
+	},
+	MaskPaths: []string{
+		"/proc/kcore",
+	},
+	ReadonlyPaths: []string{
+		"/proc/sys", "/proc/sysrq-trigger", "/proc/irq", "/proc/bus",
+	},
 	Devices:  configs.DefaultAutoCreatedDevices,
 	Hostname: "testing",
+	Mounts: []*configs.Mount{
+		{
+			Source:      "proc",
+			Destination: "/proc",
+			Device:      "proc",
+			Flags:       defaultMountFlags,
+		},
+		{
+			Source:      "tmpfs",
+			Destination: "/dev",
+			Device:      "tmpfs",
+			Flags:       syscall.MS_NOSUID | syscall.MS_STRICTATIME,
+			Data:        "mode=755",
+		},
+		{
+			Source:      "devpts",
+			Destination: "/dev/pts",
+			Device:      "devpts",
+			Flags:       syscall.MS_NOSUID | syscall.MS_NOEXEC,
+			Data:        "newinstance,ptmxmode=0666,mode=0620,gid=5",
+		},
+		{
+			Device:      "tmpfs",
+			Source:      "shm",
+			Destination: "/dev/shm",
+			Data:        "mode=1777,size=65536k",
+			Flags:       defaultMountFlags,
+		},
+		{
+			Source:      "mqueue",
+			Destination: "/dev/mqueue",
+			Device:      "mqueue",
+			Flags:       defaultMountFlags,
+		},
+		{
+			Source:      "sysfs",
+			Destination: "/sys",
+			Device:      "sysfs",
+			Flags:       defaultMountFlags | syscall.MS_RDONLY,
+		},
+	},
 	Networks: []*configs.Network{
 		{
 			Type:    "loopback",
@@ -73,8 +139,8 @@ config := &configs.Config{
 	Rlimits: []configs.Rlimit{
 		{
 			Type: syscall.RLIMIT_NOFILE,
-            Hard: uint64(1024),
+			Hard: uint64(1025),
-            Soft: uint64(1024),
+			Soft: uint64(1025),
 		},
 	},
 }
@@ -83,7 +149,11 @@ config := &configs.Config{
 Once you have the configuration populated you can create a container:
 
 ```go
-container, err := root.Create("container-id", config)
+container, err := factory.Create("container-id", config)
+if err != nil {
+	logrus.Fatal(err)
+	return
+}
 ```
 
 To spawn bash as the initial process inside the container and have the
@@ -101,13 +171,15 @@ process := &libcontainer.Process{
 
 err := container.Start(process)
 if err != nil {
-    log.Fatal(err)
+	logrus.Fatal(err)
+	container.Destroy()
+	return
 }
 
 // wait for the process to finish.
-status, err := process.Wait()
+_, err := process.Wait()
 if err != nil {
-    log.Fatal(err)
+	logrus.Fatal(err)
 }
 
 // destroy the container.
@@ -124,7 +196,6 @@ processes, err := container.Processes()
 // it's processes.
 stats, err := container.Stats()
 
-
 // pause all processes inside the container.
 container.Pause()
 

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/SPEC.md

@@ -60,7 +60,7 @@ are required to be mounted within the rootfs that the runtime will setup.
 After a container's filesystems are mounted within the newly created 
 mount namespace `/dev` will need to be populated with a set of device nodes.
 It is expected that a rootfs does not need to have any device nodes specified
-for `/dev` witin the rootfs as the container will setup the correct devices
+for `/dev` within the rootfs as the container will setup the correct devices
 that are required for executing a container's process.
 
 |      Path    | Mode |   Access   |

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/apparmor/apparmor_disabled.go

@@ -2,10 +2,19 @@
 
 package apparmor
 
+import (
+	"errors"
+)
+
+var ErrApparmorNotEnabled = errors.New("apparmor: config provided but apparmor not supported")
+
 func IsEnabled() bool {
 	return false
 }
 
 func ApplyProfile(name string) error {
+	if name != "" {
+		return ErrApparmorNotEnabled
+	}
 	return nil
 }

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/cgroups.go

@@ -15,6 +15,9 @@ type Manager interface {
 	// Returns the PIDs inside the cgroup set
 	GetPids() ([]int, error)
 
+	// Returns the PIDs inside the cgroup set & all sub-cgroups
+	GetAllPids() ([]int, error)
+
 	// Returns statistics for the cgroup set
 	GetStats() (*Stats, error)
 

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/apply_raw.go

@@ -23,6 +23,7 @@ var (
 		&MemoryGroup{},
 		&CpuGroup{},
 		&CpuacctGroup{},
+		&PidsGroup{},
 		&BlkioGroup{},
 		&HugetlbGroup{},
 		&NetClsGroup{},
@@ -112,6 +113,22 @@ func (m *Manager) Apply(pid int) (err error) {
 		return err
 	}
 
+	if c.Paths != nil {
+		paths := make(map[string]string)
+		for name, path := range c.Paths {
+			_, err := d.path(name)
+			if err != nil {
+				if cgroups.IsNotFound(err) {
+					continue
+				}
+				return err
+			}
+			paths[name] = path
+		}
+		m.Paths = paths
+		return cgroups.EnterPid(m.Paths, pid)
+	}
+
 	paths := make(map[string]string)
 	defer func() {
 		if err != nil {
@@ -135,17 +152,13 @@ func (m *Manager) Apply(pid int) (err error) {
 		paths[sys.Name()] = p
 	}
 	m.Paths = paths
-
-	if paths["cpu"] != "" {
-		if err := CheckCpushares(paths["cpu"], c.CpuShares); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
 func (m *Manager) Destroy() error {
+	if m.Cgroups.Paths != nil {
+		return nil
+	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	if err := cgroups.RemovePaths(m.Paths); err != nil {
@@ -179,15 +192,28 @@ func (m *Manager) GetStats() (*cgroups.Stats, error) {
 }
 
 func (m *Manager) Set(container *configs.Config) error {
-	for name, path := range m.Paths {
+	for _, sys := range subsystems {
-		sys, err := subsystems.Get(name)
+		// Generate fake cgroup data.
-		if err == errSubsystemDoesNotExist || !cgroups.PathExists(path) {
+		d, err := getCgroupData(container.Cgroups, -1)
-			continue
+		if err != nil {
+			return err
 		}
+		// Get the path, but don't error out if the cgroup wasn't found.
+		path, err := d.path(sys.Name())
+		if err != nil && !cgroups.IsNotFound(err) {
+			return err
+		}
+
 		if err := sys.Set(path, container.Cgroups); err != nil {
 			return err
 		}
 	}
+
+	if m.Paths["cpu"] != "" {
+		if err := CheckCpushares(m.Paths["cpu"], container.Cgroups.Resources.CpuShares); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -202,40 +228,78 @@ func (m *Manager) Freeze(state configs.FreezerState) error {
 	if err != nil {
 		return err
 	}
-	prevState := m.Cgroups.Freezer
+	prevState := m.Cgroups.Resources.Freezer
-	m.Cgroups.Freezer = state
+	m.Cgroups.Resources.Freezer = state
 	freezer, err := subsystems.Get("freezer")
 	if err != nil {
 		return err
 	}
 	err = freezer.Set(dir, m.Cgroups)
 	if err != nil {
-		m.Cgroups.Freezer = prevState
+		m.Cgroups.Resources.Freezer = prevState
 		return err
 	}
 	return nil
 }
 
 func (m *Manager) GetPids() ([]int, error) {
-	d, err := getCgroupData(m.Cgroups, 0)
+	dir, err := getCgroupPath(m.Cgroups)
 	if err != nil {
 		return nil, err
 	}
-
-	dir, err := d.path("devices")
-	if err != nil {
-		return nil, err
-	}
-
 	return cgroups.GetPids(dir)
 }
 
+func (m *Manager) GetAllPids() ([]int, error) {
+	dir, err := getCgroupPath(m.Cgroups)
+	if err != nil {
+		return nil, err
+	}
+	return cgroups.GetAllPids(dir)
+}
+
+func getCgroupPath(c *configs.Cgroup) (string, error) {
+	d, err := getCgroupData(c, 0)
+	if err != nil {
+		return "", err
+	}
+
+	return d.path("devices")
+}
+
+// pathClean makes a path safe for use with filepath.Join. This is done by not
+// only cleaning the path, but also (if the path is relative) adding a leading
+// '/' and cleaning it (then removing the leading '/'). This ensures that a
+// path resulting from prepending another path will always resolve to lexically
+// be a subdirectory of the prefixed path. This is all done lexically, so paths
+// that include symlinks won't be safe as a result of using pathClean.
+func pathClean(path string) string {
+	// Ensure that all paths are cleaned (especially problematic ones like
+	// "/../../../../../" which can cause lots of issues).
+	path = filepath.Clean(path)
+
+	// If the path isn't absolute, we need to do more processing to fix paths
+	// such as "../../../../<etc>/some/path". We also shouldn't convert absolute
+	// paths to relative ones.
+	if !filepath.IsAbs(path) {
+		path = filepath.Clean(string(os.PathSeparator) + path)
+		// This can't fail, as (by definition) all paths are relative to root.
+		path, _ = filepath.Rel(string(os.PathSeparator), path)
+	}
+
+	// Clean the path again for good measure.
+	return filepath.Clean(path)
+}
+
 func getCgroupData(c *configs.Cgroup, pid int) (*cgroupData, error) {
 	root, err := getCgroupRoot()
 	if err != nil {
 		return nil, err
 	}
 
+	// Clean the parent slice path.
+	c.Parent = pathClean(c.Parent)
+
 	return &cgroupData{
 		root:   root,
 		parent: c.Parent,

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/blkio.go

@@ -22,31 +22,26 @@ func (s *BlkioGroup) Name() string {
 }
 
 func (s *BlkioGroup) Apply(d *cgroupData) error {
-	dir, err := d.join("blkio")
+	_, err := d.join("blkio")
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-
-	if err := s.Set(dir, d.config); err != nil {
-		return err
-	}
-
 	return nil
 }
 
 func (s *BlkioGroup) Set(path string, cgroup *configs.Cgroup) error {
-	if cgroup.BlkioWeight != 0 {
+	if cgroup.Resources.BlkioWeight != 0 {
-		if err := writeFile(path, "blkio.weight", strconv.FormatUint(uint64(cgroup.BlkioWeight), 10)); err != nil {
+		if err := writeFile(path, "blkio.weight", strconv.FormatUint(uint64(cgroup.Resources.BlkioWeight), 10)); err != nil {
 			return err
 		}
 	}
 
-	if cgroup.BlkioLeafWeight != 0 {
+	if cgroup.Resources.BlkioLeafWeight != 0 {
-		if err := writeFile(path, "blkio.leaf_weight", strconv.FormatUint(uint64(cgroup.BlkioLeafWeight), 10)); err != nil {
+		if err := writeFile(path, "blkio.leaf_weight", strconv.FormatUint(uint64(cgroup.Resources.BlkioLeafWeight), 10)); err != nil {
 			return err
 		}
 	}
-	for _, wd := range cgroup.BlkioWeightDevice {
+	for _, wd := range cgroup.Resources.BlkioWeightDevice {
 		if err := writeFile(path, "blkio.weight_device", wd.WeightString()); err != nil {
 			return err
 		}
@@ -54,22 +49,22 @@ func (s *BlkioGroup) Set(path string, cgroup *configs.Cgroup) error {
 			return err
 		}
 	}
-	for _, td := range cgroup.BlkioThrottleReadBpsDevice {
+	for _, td := range cgroup.Resources.BlkioThrottleReadBpsDevice {
 		if err := writeFile(path, "blkio.throttle.read_bps_device", td.String()); err != nil {
 			return err
 		}
 	}
-	for _, td := range cgroup.BlkioThrottleWriteBpsDevice {
+	for _, td := range cgroup.Resources.BlkioThrottleWriteBpsDevice {
 		if err := writeFile(path, "blkio.throttle.write_bps_device", td.String()); err != nil {
 			return err
 		}
 	}
-	for _, td := range cgroup.BlkioThrottleReadIOPSDevice {
+	for _, td := range cgroup.Resources.BlkioThrottleReadIOPSDevice {
 		if err := writeFile(path, "blkio.throttle.read_iops_device", td.String()); err != nil {
 			return err
 		}
 	}
-	for _, td := range cgroup.BlkioThrottleWriteIOPSDevice {
+	for _, td := range cgroup.Resources.BlkioThrottleWriteIOPSDevice {
 		if err := writeFile(path, "blkio.throttle.write_iops_device", td.String()); err != nil {
 			return err
 		}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpu.go

@@ -22,41 +22,36 @@ func (s *CpuGroup) Name() string {
 func (s *CpuGroup) Apply(d *cgroupData) error {
 	// We always want to join the cpu group, to allow fair cpu scheduling
 	// on a container basis
-	dir, err := d.join("cpu")
+	_, err := d.join("cpu")
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-
-	if err := s.Set(dir, d.config); err != nil {
-		return err
-	}
-
 	return nil
 }
 
 func (s *CpuGroup) Set(path string, cgroup *configs.Cgroup) error {
-	if cgroup.CpuShares != 0 {
+	if cgroup.Resources.CpuShares != 0 {
-		if err := writeFile(path, "cpu.shares", strconv.FormatInt(cgroup.CpuShares, 10)); err != nil {
+		if err := writeFile(path, "cpu.shares", strconv.FormatInt(cgroup.Resources.CpuShares, 10)); err != nil {
 			return err
 		}
 	}
-	if cgroup.CpuPeriod != 0 {
+	if cgroup.Resources.CpuPeriod != 0 {
-		if err := writeFile(path, "cpu.cfs_period_us", strconv.FormatInt(cgroup.CpuPeriod, 10)); err != nil {
+		if err := writeFile(path, "cpu.cfs_period_us", strconv.FormatInt(cgroup.Resources.CpuPeriod, 10)); err != nil {
 			return err
 		}
 	}
-	if cgroup.CpuQuota != 0 {
+	if cgroup.Resources.CpuQuota != 0 {
-		if err := writeFile(path, "cpu.cfs_quota_us", strconv.FormatInt(cgroup.CpuQuota, 10)); err != nil {
+		if err := writeFile(path, "cpu.cfs_quota_us", strconv.FormatInt(cgroup.Resources.CpuQuota, 10)); err != nil {
 			return err
 		}
 	}
-	if cgroup.CpuRtPeriod != 0 {
+	if cgroup.Resources.CpuRtPeriod != 0 {
-		if err := writeFile(path, "cpu.rt_period_us", strconv.FormatInt(cgroup.CpuRtPeriod, 10)); err != nil {
+		if err := writeFile(path, "cpu.rt_period_us", strconv.FormatInt(cgroup.Resources.CpuRtPeriod, 10)); err != nil {
 			return err
 		}
 	}
-	if cgroup.CpuRtRuntime != 0 {
+	if cgroup.Resources.CpuRtRuntime != 0 {
-		if err := writeFile(path, "cpu.rt_runtime_us", strconv.FormatInt(cgroup.CpuRtRuntime, 10)); err != nil {
+		if err := writeFile(path, "cpu.rt_runtime_us", strconv.FormatInt(cgroup.Resources.CpuRtRuntime, 10)); err != nil {
 			return err
 		}
 	}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/cpuset.go

@@ -4,6 +4,7 @@ package fs
 
 import (
 	"bytes"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -29,13 +30,13 @@ func (s *CpusetGroup) Apply(d *cgroupData) error {
 }
 
 func (s *CpusetGroup) Set(path string, cgroup *configs.Cgroup) error {
-	if cgroup.CpusetCpus != "" {
+	if cgroup.Resources.CpusetCpus != "" {
-		if err := writeFile(path, "cpuset.cpus", cgroup.CpusetCpus); err != nil {
+		if err := writeFile(path, "cpuset.cpus", cgroup.Resources.CpusetCpus); err != nil {
 			return err
 		}
 	}
-	if cgroup.CpusetMems != "" {
+	if cgroup.Resources.CpusetMems != "" {
-		if err := writeFile(path, "cpuset.mems", cgroup.CpusetMems); err != nil {
+		if err := writeFile(path, "cpuset.mems", cgroup.Resources.CpusetMems); err != nil {
 			return err
 		}
 	}
@@ -63,11 +64,6 @@ func (s *CpusetGroup) ApplyDir(dir string, cgroup *configs.Cgroup, pid int) erro
 	if err := s.ensureParent(dir, root); err != nil {
 		return err
 	}
-	// the default values inherit from parent cgroup are already set in
-	// s.ensureParent, cover these if we have our own
-	if err := s.Set(dir, cgroup); err != nil {
-		return err
-	}
 	// because we are not using d.join we need to place the pid into the procs file
 	// unlike the other subsystems
 	if err := writeFile(dir, "cgroup.procs", strconv.Itoa(pid)); err != nil {
@@ -95,6 +91,10 @@ func (s *CpusetGroup) ensureParent(current, root string) error {
 	if filepath.Clean(parent) == root {
 		return nil
 	}
+	// Avoid infinite recursion.
+	if parent == current {
+		return fmt.Errorf("cpuset: cgroup parent path outside cgroup root")
+	}
 	if err := s.ensureParent(parent, root); err != nil {
 		return err
 	}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/devices.go

@@ -15,27 +15,22 @@ func (s *DevicesGroup) Name() string {
 }
 
 func (s *DevicesGroup) Apply(d *cgroupData) error {
-	dir, err := d.join("devices")
+	_, err := d.join("devices")
 	if err != nil {
 		// We will return error even it's `not found` error, devices
 		// cgroup is hard requirement for container's security.
 		return err
 	}
-
-	if err := s.Set(dir, d.config); err != nil {
-		return err
-	}
-
 	return nil
 }
 
 func (s *DevicesGroup) Set(path string, cgroup *configs.Cgroup) error {
-	if !cgroup.AllowAllDevices {
+	if !cgroup.Resources.AllowAllDevices {
 		if err := writeFile(path, "devices.deny", "a"); err != nil {
 			return err
 		}
 
-		for _, dev := range cgroup.AllowedDevices {
+		for _, dev := range cgroup.Resources.AllowedDevices {
 			if err := writeFile(path, "devices.allow", dev.CgroupString()); err != nil {
 				return err
 			}
@@ -47,7 +42,7 @@ func (s *DevicesGroup) Set(path string, cgroup *configs.Cgroup) error {
 		return err
 	}
 
-	for _, dev := range cgroup.DeniedDevices {
+	for _, dev := range cgroup.Resources.DeniedDevices {
 		if err := writeFile(path, "devices.deny", dev.CgroupString()); err != nil {
 			return err
 		}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/freezer.go

@@ -19,22 +19,17 @@ func (s *FreezerGroup) Name() string {
 }
 
 func (s *FreezerGroup) Apply(d *cgroupData) error {
-	dir, err := d.join("freezer")
+	_, err := d.join("freezer")
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-
-	if err := s.Set(dir, d.config); err != nil {
-		return err
-	}
-
 	return nil
 }
 
 func (s *FreezerGroup) Set(path string, cgroup *configs.Cgroup) error {
-	switch cgroup.Freezer {
+	switch cgroup.Resources.Freezer {
 	case configs.Frozen, configs.Thawed:
-		if err := writeFile(path, "freezer.state", string(cgroup.Freezer)); err != nil {
+		if err := writeFile(path, "freezer.state", string(cgroup.Resources.Freezer)); err != nil {
 			return err
 		}
 
@@ -43,7 +38,7 @@ func (s *FreezerGroup) Set(path string, cgroup *configs.Cgroup) error {
 			if err != nil {
 				return err
 			}
-			if strings.TrimSpace(state) == string(cgroup.Freezer) {
+			if strings.TrimSpace(state) == string(cgroup.Resources.Freezer) {
 				break
 			}
 			time.Sleep(1 * time.Millisecond)
@@ -51,7 +46,7 @@ func (s *FreezerGroup) Set(path string, cgroup *configs.Cgroup) error {
 	case configs.Undefined:
 		return nil
 	default:
-		return fmt.Errorf("Invalid argument '%s' to freezer.state", string(cgroup.Freezer))
+		return fmt.Errorf("Invalid argument '%s' to freezer.state", string(cgroup.Resources.Freezer))
 	}
 
 	return nil

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/hugetlb.go

@@ -19,20 +19,15 @@ func (s *HugetlbGroup) Name() string {
 }
 
 func (s *HugetlbGroup) Apply(d *cgroupData) error {
-	dir, err := d.join("hugetlb")
+	_, err := d.join("hugetlb")
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-
-	if err := s.Set(dir, d.config); err != nil {
-		return err
-	}
-
 	return nil
 }
 
 func (s *HugetlbGroup) Set(path string, cgroup *configs.Cgroup) error {
-	for _, hugetlb := range cgroup.HugetlbLimit {
+	for _, hugetlb := range cgroup.Resources.HugetlbLimit {
 		if err := writeFile(path, strings.Join([]string{"hugetlb", hugetlb.Pagesize, "limit_in_bytes"}, "."), strconv.FormatUint(hugetlb.Limit, 10)); err != nil {
 			return err
 		}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/memory.go

@@ -32,8 +32,9 @@ func (s *MemoryGroup) Apply(d *cgroupData) (err error) {
 				return err
 			}
 		}
-
+		// We have to set kernel memory here, as we can't change it once
-		if err := s.Set(path, d.config); err != nil {
+		// processes have been attached.
+		if err := s.SetKernelMemory(path, d.config); err != nil {
 			return err
 		}
 	}
@@ -50,45 +51,49 @@ func (s *MemoryGroup) Apply(d *cgroupData) (err error) {
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
+	return nil
+}
 
+func (s *MemoryGroup) SetKernelMemory(path string, cgroup *configs.Cgroup) error {
+	// This has to be done separately because it has special constraints (it
+	// can't be done after there are processes attached to the cgroup).
+	if cgroup.Resources.KernelMemory > 0 {
+		if err := writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemory, 10)); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
 func (s *MemoryGroup) Set(path string, cgroup *configs.Cgroup) error {
-	if cgroup.Memory != 0 {
+	if cgroup.Resources.Memory != 0 {
-		if err := writeFile(path, "memory.limit_in_bytes", strconv.FormatInt(cgroup.Memory, 10)); err != nil {
+		if err := writeFile(path, "memory.limit_in_bytes", strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil {
 			return err
 		}
 	}
-	if cgroup.MemoryReservation != 0 {
+	if cgroup.Resources.MemoryReservation != 0 {
-		if err := writeFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(cgroup.MemoryReservation, 10)); err != nil {
+		if err := writeFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil {
 			return err
 		}
 	}
-	if cgroup.MemorySwap > 0 {
+	if cgroup.Resources.MemorySwap > 0 {
-		if err := writeFile(path, "memory.memsw.limit_in_bytes", strconv.FormatInt(cgroup.MemorySwap, 10)); err != nil {
+		if err := writeFile(path, "memory.memsw.limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil {
 			return err
 		}
 	}
-	if cgroup.KernelMemory > 0 {
+	if cgroup.Resources.OomKillDisable {
-		if err := writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(cgroup.KernelMemory, 10)); err != nil {
-			return err
-		}
-	}
-
-	if cgroup.OomKillDisable {
 		if err := writeFile(path, "memory.oom_control", "1"); err != nil {
 			return err
 		}
 	}
-	if cgroup.MemorySwappiness >= 0 && cgroup.MemorySwappiness <= 100 {
+	if cgroup.Resources.MemorySwappiness >= 0 && cgroup.Resources.MemorySwappiness <= 100 {
-		if err := writeFile(path, "memory.swappiness", strconv.FormatInt(cgroup.MemorySwappiness, 10)); err != nil {
+		if err := writeFile(path, "memory.swappiness", strconv.FormatInt(cgroup.Resources.MemorySwappiness, 10)); err != nil {
 			return err
 		}
-	} else if cgroup.MemorySwappiness == -1 {
+	} else if cgroup.Resources.MemorySwappiness == -1 {
 		return nil
 	} else {
-		return fmt.Errorf("invalid value:%d. valid memory swappiness range is 0-100", cgroup.MemorySwappiness)
+		return fmt.Errorf("invalid value:%d. valid memory swappiness range is 0-100", cgroup.Resources.MemorySwappiness)
 	}
 
 	return nil
@@ -139,12 +144,12 @@ func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
 }
 
 func memoryAssigned(cgroup *configs.Cgroup) bool {
-	return cgroup.Memory != 0 ||
+	return cgroup.Resources.Memory != 0 ||
-		cgroup.MemoryReservation != 0 ||
+		cgroup.Resources.MemoryReservation != 0 ||
-		cgroup.MemorySwap > 0 ||
+		cgroup.Resources.MemorySwap > 0 ||
-		cgroup.KernelMemory > 0 ||
+		cgroup.Resources.KernelMemory > 0 ||
-		cgroup.OomKillDisable ||
+		cgroup.Resources.OomKillDisable ||
-		cgroup.MemorySwappiness != -1
+		cgroup.Resources.MemorySwappiness != -1
 }
 
 func getMemoryData(path, name string) (cgroups.MemoryData, error) {

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_cls.go

@@ -15,21 +15,16 @@ func (s *NetClsGroup) Name() string {
 }
 
 func (s *NetClsGroup) Apply(d *cgroupData) error {
-	dir, err := d.join("net_cls")
+	_, err := d.join("net_cls")
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-
-	if err := s.Set(dir, d.config); err != nil {
-		return err
-	}
-
 	return nil
 }
 
 func (s *NetClsGroup) Set(path string, cgroup *configs.Cgroup) error {
-	if cgroup.NetClsClassid != "" {
+	if cgroup.Resources.NetClsClassid != "" {
-		if err := writeFile(path, "net_cls.classid", cgroup.NetClsClassid); err != nil {
+		if err := writeFile(path, "net_cls.classid", cgroup.Resources.NetClsClassid); err != nil {
 			return err
 		}
 	}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/net_prio.go

@@ -15,20 +15,15 @@ func (s *NetPrioGroup) Name() string {
 }
 
 func (s *NetPrioGroup) Apply(d *cgroupData) error {
-	dir, err := d.join("net_prio")
+	_, err := d.join("net_prio")
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-
-	if err := s.Set(dir, d.config); err != nil {
-		return err
-	}
-
 	return nil
 }
 
 func (s *NetPrioGroup) Set(path string, cgroup *configs.Cgroup) error {
-	for _, prioMap := range cgroup.NetPrioIfpriomap {
+	for _, prioMap := range cgroup.Resources.NetPrioIfpriomap {
 		if err := writeFile(path, "net_prio.ifpriomap", prioMap.CgroupString()); err != nil {
 			return err
 		}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/fs/pids.go

@@ -0,0 +1,57 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type PidsGroup struct {
+}
+
+func (s *PidsGroup) Name() string {
+	return "pids"
+}
+
+func (s *PidsGroup) Apply(d *cgroupData) error {
+	_, err := d.join("pids")
+	if err != nil && !cgroups.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func (s *PidsGroup) Set(path string, cgroup *configs.Cgroup) error {
+	if cgroup.Resources.PidsLimit != 0 {
+		// "max" is the fallback value.
+		limit := "max"
+
+		if cgroup.Resources.PidsLimit > 0 {
+			limit = strconv.FormatInt(cgroup.Resources.PidsLimit, 10)
+		}
+
+		if err := writeFile(path, "pids.max", limit); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *PidsGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("pids"))
+}
+
+func (s *PidsGroup) GetStats(path string, stats *cgroups.Stats) error {
+	value, err := getCgroupParamUint(path, "pids.current")
+	if err != nil {
+		return fmt.Errorf("failed to parse pids.current - %s", err)
+	}
+
+	stats.PidsStats.Current = value
+	return nil
+}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/stats.go

@@ -49,6 +49,11 @@ type MemoryStats struct {
 	Stats       map[string]uint64 `json:"stats,omitempty"`
 }
 
+type PidsStats struct {
+	// number of pids in the cgroup
+	Current uint64 `json:"current,omitempty"`
+}
+
 type BlkioStatEntry struct {
 	Major uint64 `json:"major,omitempty"`
 	Minor uint64 `json:"minor,omitempty"`
@@ -80,6 +85,7 @@ type HugetlbStats struct {
 type Stats struct {
 	CpuStats    CpuStats    `json:"cpu_stats,omitempty"`
 	MemoryStats MemoryStats `json:"memory_stats,omitempty"`
+	PidsStats   PidsStats   `json:"pids_stats,omitempty"`
 	BlkioStats  BlkioStats  `json:"blkio_stats,omitempty"`
 	// the map is in the format "size of hugepage: stats of the hugepage"
 	HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"`

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_nosystemd.go

@@ -26,6 +26,10 @@ func (m *Manager) GetPids() ([]int, error) {
 	return nil, fmt.Errorf("Systemd not supported")
 }
 
+func (m *Manager) GetAllPids() ([]int, error) {
+	return nil, fmt.Errorf("Systemd not supported")
+}
+
 func (m *Manager) Destroy() error {
 	return fmt.Errorf("Systemd not supported")
 }

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/systemd/apply_systemd.go

@@ -55,6 +55,7 @@ var subsystems = subsystemSet{
 	&fs.MemoryGroup{},
 	&fs.CpuGroup{},
 	&fs.CpuacctGroup{},
+	&fs.PidsGroup{},
 	&fs.BlkioGroup{},
 	&fs.HugetlbGroup{},
 	&fs.PerfEventGroup{},
@@ -167,8 +168,25 @@ func (m *Manager) Apply(pid int) error {
 		properties []systemdDbus.Property
 	)
 
-	if c.Slice != "" {
+	if c.Paths != nil {
-		slice = c.Slice
+		paths := make(map[string]string)
+		for name, path := range c.Paths {
+			_, err := getSubsystemPath(m.Cgroups, name)
+			if err != nil {
+				// Don't fail if a cgroup hierarchy was not found, just skip this subsystem
+				if cgroups.IsNotFound(err) {
+					continue
+				}
+				return err
+			}
+			paths[name] = path
+		}
+		m.Paths = paths
+		return cgroups.EnterPid(m.Paths, pid)
+	}
+
+	if c.Parent != "" {
+		slice = c.Parent
 	}
 
 	properties = append(properties,
@@ -189,26 +207,26 @@ func (m *Manager) Apply(pid int) error {
 			newProp("DefaultDependencies", false))
 	}
 
-	if c.Memory != 0 {
+	if c.Resources.Memory != 0 {
 		properties = append(properties,
-			newProp("MemoryLimit", uint64(c.Memory)))
+			newProp("MemoryLimit", uint64(c.Resources.Memory)))
 	}
 
-	if c.CpuShares != 0 {
+	if c.Resources.CpuShares != 0 {
 		properties = append(properties,
-			newProp("CPUShares", uint64(c.CpuShares)))
+			newProp("CPUShares", uint64(c.Resources.CpuShares)))
 	}
 
-	if c.BlkioWeight != 0 {
+	if c.Resources.BlkioWeight != 0 {
 		properties = append(properties,
-			newProp("BlockIOWeight", uint64(c.BlkioWeight)))
+			newProp("BlockIOWeight", uint64(c.Resources.BlkioWeight)))
 	}
 
 	// We need to set kernel memory before processes join cgroup because
 	// kmem.limit_in_bytes can only be set when the cgroup is empty.
 	// And swap memory limit needs to be set after memory limit, only
 	// memory limit is handled by systemd, so it's kind of ugly here.
-	if c.KernelMemory > 0 {
+	if c.Resources.KernelMemory > 0 {
 		if err := setKernelMemory(c); err != nil {
 			return err
 		}
@@ -233,7 +251,7 @@ func (m *Manager) Apply(pid int) error {
 		return err
 	}
 
-	// we need to manually join the freezer, net_cls, net_prio and cpuset cgroup in systemd
+	// we need to manually join the freezer, net_cls, net_prio, pids and cpuset cgroup in systemd
 	// because it does not currently support it via the dbus api.
 	if err := joinFreezer(c, pid); err != nil {
 		return err
@@ -246,6 +264,10 @@ func (m *Manager) Apply(pid int) error {
 		return err
 	}
 
+	if err := joinPids(c, pid); err != nil {
+		return err
+	}
+
 	if err := joinCpuset(c, pid); err != nil {
 		return err
 	}
@@ -277,17 +299,13 @@ func (m *Manager) Apply(pid int) error {
 		paths[s.Name()] = subsystemPath
 	}
 	m.Paths = paths
-
-	if paths["cpu"] != "" {
-		if err := fs.CheckCpushares(paths["cpu"], c.CpuShares); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
 func (m *Manager) Destroy() error {
+	if m.Cgroups.Paths != nil {
+		return nil
+	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	theConn.StopUnit(getUnitName(m.Cgroups), "replace", nil)
@@ -330,68 +348,43 @@ func join(c *configs.Cgroup, subsystem string, pid int) (string, error) {
 }
 
 func joinCpu(c *configs.Cgroup, pid int) error {
-	path, err := getSubsystemPath(c, "cpu")
+	_, err := join(c, "cpu", pid)
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-	if c.CpuQuota != 0 {
-		if err = writeFile(path, "cpu.cfs_quota_us", strconv.FormatInt(c.CpuQuota, 10)); err != nil {
-			return err
-		}
-	}
-	if c.CpuPeriod != 0 {
-		if err = writeFile(path, "cpu.cfs_period_us", strconv.FormatInt(c.CpuPeriod, 10)); err != nil {
-			return err
-		}
-	}
-	if c.CpuRtPeriod != 0 {
-		if err = writeFile(path, "cpu.rt_period_us", strconv.FormatInt(c.CpuRtPeriod, 10)); err != nil {
-			return err
-		}
-	}
-	if c.CpuRtRuntime != 0 {
-		if err = writeFile(path, "cpu.rt_runtime_us", strconv.FormatInt(c.CpuRtRuntime, 10)); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
 func joinFreezer(c *configs.Cgroup, pid int) error {
-	path, err := join(c, "freezer", pid)
+	_, err := join(c, "freezer", pid)
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-	freezer, err := subsystems.Get("freezer")
+	return nil
-	if err != nil {
-		return err
-	}
-	return freezer.Set(path, c)
 }
 
 func joinNetPrio(c *configs.Cgroup, pid int) error {
-	path, err := join(c, "net_prio", pid)
+	_, err := join(c, "net_prio", pid)
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-	netPrio, err := subsystems.Get("net_prio")
+	return nil
-	if err != nil {
-		return err
-	}
-	return netPrio.Set(path, c)
 }
 
 func joinNetCls(c *configs.Cgroup, pid int) error {
-	path, err := join(c, "net_cls", pid)
+	_, err := join(c, "net_cls", pid)
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-	netcls, err := subsystems.Get("net_cls")
+	return nil
-	if err != nil {
+}
+
+func joinPids(c *configs.Cgroup, pid int) error {
+	_, err := join(c, "pids", pid)
+	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-	return netcls.Set(path, c)
+	return nil
 }
 
 func getSubsystemPath(c *configs.Cgroup, subsystem string) (string, error) {
@@ -406,8 +399,8 @@ func getSubsystemPath(c *configs.Cgroup, subsystem string) (string, error) {
 	}
 
 	slice := "system.slice"
-	if c.Slice != "" {
+	if c.Parent != "" {
-		slice = c.Slice
+		slice = c.Parent
 	}
 
 	return filepath.Join(mountpoint, initPath, slice, getUnitName(c)), nil
@@ -418,15 +411,15 @@ func (m *Manager) Freeze(state configs.FreezerState) error {
 	if err != nil {
 		return err
 	}
-	prevState := m.Cgroups.Freezer
+	prevState := m.Cgroups.Resources.Freezer
-	m.Cgroups.Freezer = state
+	m.Cgroups.Resources.Freezer = state
 	freezer, err := subsystems.Get("freezer")
 	if err != nil {
 		return err
 	}
 	err = freezer.Set(path, m.Cgroups)
 	if err != nil {
-		m.Cgroups.Freezer = prevState
+		m.Cgroups.Resources.Freezer = prevState
 		return err
 	}
 	return nil
@@ -440,6 +433,14 @@ func (m *Manager) GetPids() ([]int, error) {
 	return cgroups.GetPids(path)
 }
 
+func (m *Manager) GetAllPids() ([]int, error) {
+	path, err := getSubsystemPath(m.Cgroups, "devices")
+	if err != nil {
+		return nil, err
+	}
+	return cgroups.GetAllPids(path)
+}
+
 func (m *Manager) GetStats() (*cgroups.Stats, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -458,21 +459,28 @@ func (m *Manager) GetStats() (*cgroups.Stats, error) {
 }
 
 func (m *Manager) Set(container *configs.Config) error {
-	for name, path := range m.Paths {
+	for _, sys := range subsystems {
-		sys, err := subsystems.Get(name)
+		// Get the subsystem path, but don't error out for not found cgroups.
-		if err == errSubsystemDoesNotExist || !cgroups.PathExists(path) {
+		path, err := getSubsystemPath(container.Cgroups, sys.Name())
-			continue
+		if err != nil && !cgroups.IsNotFound(err) {
+			return err
 		}
+
 		if err := sys.Set(path, container.Cgroups); err != nil {
 			return err
 		}
 	}
 
+	if m.Paths["cpu"] != "" {
+		if err := fs.CheckCpushares(m.Paths["cpu"], container.Cgroups.Resources.CpuShares); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
 func getUnitName(c *configs.Cgroup) string {
-	return fmt.Sprintf("%s-%s.scope", c.Parent, c.Name)
+	return fmt.Sprintf("%s-%s.scope", c.ScopePrefix, c.Name)
 }
 
 // Atm we can't use the systemd device support because of two missing things:
@@ -487,17 +495,13 @@ func getUnitName(c *configs.Cgroup) string {
 // because systemd will re-write the device settings if it needs to re-apply the cgroup context.
 // This happens at least for v208 when any sibling unit is started.
 func joinDevices(c *configs.Cgroup, pid int) error {
-	path, err := join(c, "devices", pid)
+	_, err := join(c, "devices", pid)
 	// Even if it's `not found` error, we'll return err because devices cgroup
 	// is hard requirement for container security.
 	if err != nil {
 		return err
 	}
-	devices, err := subsystems.Get("devices")
+	return nil
-	if err != nil {
-		return err
-	}
-	return devices.Set(path, c)
 }
 
 func setKernelMemory(c *configs.Cgroup) error {
@@ -510,52 +514,16 @@ func setKernelMemory(c *configs.Cgroup) error {
 		return err
 	}
 
-	if c.KernelMemory > 0 {
+	// This doesn't get called by manager.Set, so we need to do it here.
-		err = writeFile(path, "memory.kmem.limit_in_bytes", strconv.FormatInt(c.KernelMemory, 10))
+	s := &fs.MemoryGroup{}
-		if err != nil {
+	return s.SetKernelMemory(path, c)
-			return err
-		}
-	}
-
-	return nil
 }
 
 func joinMemory(c *configs.Cgroup, pid int) error {
-	path, err := getSubsystemPath(c, "memory")
+	_, err := join(c, "memory", pid)
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-
-	// -1 disables memoryswap
-	if c.MemorySwap > 0 {
-		err = writeFile(path, "memory.memsw.limit_in_bytes", strconv.FormatInt(c.MemorySwap, 10))
-		if err != nil {
-			return err
-		}
-	}
-	if c.MemoryReservation > 0 {
-		err = writeFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(c.MemoryReservation, 10))
-		if err != nil {
-			return err
-		}
-	}
-	if c.OomKillDisable {
-		if err := writeFile(path, "memory.oom_control", "1"); err != nil {
-			return err
-		}
-	}
-
-	if c.MemorySwappiness >= 0 && c.MemorySwappiness <= 100 {
-		err = writeFile(path, "memory.swappiness", strconv.FormatInt(c.MemorySwappiness, 10))
-		if err != nil {
-			return err
-		}
-	} else if c.MemorySwappiness == -1 {
-		return nil
-	} else {
-		return fmt.Errorf("invalid value:%d. valid memory swappiness range is 0-100", c.MemorySwappiness)
-	}
-
 	return nil
 }
 
@@ -577,68 +545,25 @@ func joinCpuset(c *configs.Cgroup, pid int) error {
 // expects device path instead of major minor numbers, which is also confusing
 // for users. So we use fs work around for now.
 func joinBlkio(c *configs.Cgroup, pid int) error {
-	path, err := getSubsystemPath(c, "blkio")
+	_, err := join(c, "blkio", pid)
 	if err != nil {
 		return err
 	}
-	// systemd doesn't directly support this in the dbus properties
-	if c.BlkioLeafWeight != 0 {
-		if err := writeFile(path, "blkio.leaf_weight", strconv.FormatUint(uint64(c.BlkioLeafWeight), 10)); err != nil {
-			return err
-		}
-	}
-	for _, wd := range c.BlkioWeightDevice {
-		if err := writeFile(path, "blkio.weight_device", wd.WeightString()); err != nil {
-			return err
-		}
-		if err := writeFile(path, "blkio.leaf_weight_device", wd.LeafWeightString()); err != nil {
-			return err
-		}
-	}
-	for _, td := range c.BlkioThrottleReadBpsDevice {
-		if err := writeFile(path, "blkio.throttle.read_bps_device", td.String()); err != nil {
-			return err
-		}
-	}
-	for _, td := range c.BlkioThrottleWriteBpsDevice {
-		if err := writeFile(path, "blkio.throttle.write_bps_device", td.String()); err != nil {
-			return err
-		}
-	}
-	for _, td := range c.BlkioThrottleReadIOPSDevice {
-		if err := writeFile(path, "blkio.throttle.read_iops_device", td.String()); err != nil {
-			return err
-		}
-	}
-	for _, td := range c.BlkioThrottleWriteIOPSDevice {
-		if err := writeFile(path, "blkio.throttle.write_iops_device", td.String()); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
 func joinHugetlb(c *configs.Cgroup, pid int) error {
-	path, err := join(c, "hugetlb", pid)
+	_, err := join(c, "hugetlb", pid)
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-	hugetlb, err := subsystems.Get("hugetlb")
+	return nil
-	if err != nil {
-		return err
-	}
-	return hugetlb.Set(path, c)
 }
 
 func joinPerfEvent(c *configs.Cgroup, pid int) error {
-	path, err := join(c, "perf_event", pid)
+	_, err := join(c, "perf_event", pid)
 	if err != nil && !cgroups.IsNotFound(err) {
 		return err
 	}
-	perfEvent, err := subsystems.Get("perf_event")
+	return nil
-	if err != nil {
-		return err
-	}
-	return perfEvent.Set(path, c)
 }

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/cgroups/utils.go

@@ -13,7 +13,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/pkg/mount"
-	"github.com/docker/docker/pkg/units"
+	"github.com/docker/go-units"
 )
 
 const cgroupNamePrefix = "name="
@@ -84,10 +84,19 @@ func FindCgroupMountpointDir() (string, error) {
 		// Safe as mountinfo encodes mountpoints with spaces as \040.
 		index := strings.Index(text, " - ")
 		postSeparatorFields := strings.Fields(text[index+3:])
-		if len(postSeparatorFields) < 3 {
+		numPostFields := len(postSeparatorFields)
+
+		// This is an error as we can't detect if the mount is for "cgroup"
+		if numPostFields == 0 {
+			return "", fmt.Errorf("Found no fields post '-' in %q", text)
+		}
+
+		if postSeparatorFields[0] == "cgroup" {
+			// Check that the mount is properly formated.
+			if numPostFields < 3 {
 				return "", fmt.Errorf("Error found less than 3 fields post '-' in %q", text)
 			}
-		if postSeparatorFields[0] == "cgroup" {
+
 			return filepath.Dir(fields[4]), nil
 		}
 	}
@@ -323,9 +332,14 @@ func GetHugePageSize() ([]string, error) {
 	return pageSizes, nil
 }
 
-// GetPids returns all pids, that were added to cgroup at path and to all its
+// GetPids returns all pids, that were added to cgroup at path.
-// subcgroups.
 func GetPids(path string) ([]int, error) {
+	return readProcsFile(path)
+}
+
+// GetAllPids returns all pids, that were added to cgroup at path and to all its
+// subcgroups.
+func GetAllPids(path string) ([]int, error) {
 	var pids []int
 	// collect pids from all sub-cgroups
 	err := filepath.Walk(path, func(p string, info os.FileInfo, iErr error) error {

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/configs/cgroup_unix.go

@@ -16,6 +16,17 @@ type Cgroup struct {
 	// name of parent cgroup or slice
 	Parent string `json:"parent"`
 
+	// ScopePrefix decribes prefix for the scope name
+	ScopePrefix string `json:"scope_prefix"`
+
+	// Paths represent the cgroups paths to join
+	Paths map[string]string
+
+	// Resources contains various cgroups settings to apply
+	*Resources
+}
+
+type Resources struct {
 	// If this is true allow access to any kind of device within the container.  If false, allow access only to devices explicitly listed in the allowed_devices list.
 	AllowAllDevices bool `json:"allow_all_devices"`
 
@@ -29,7 +40,7 @@ type Cgroup struct {
 	// Memory reservation or soft_limit (in bytes)
 	MemoryReservation int64 `json:"memory_reservation"`
 
-	// Total memory usage (memory + swap); set `-1' to disable swap
+	// Total memory usage (memory + swap); set `-1` to enable unlimited swap
 	MemorySwap int64 `json:"memory_swap"`
 
 	// Kernel memory limit (in bytes)
@@ -56,6 +67,9 @@ type Cgroup struct {
 	// MEM to use
 	CpusetMems string `json:"cpuset_mems"`
 
+	// Process limit; set <= `0' to disable limit.
+	PidsLimit int64 `json:"pids_limit"`
+
 	// Specifies per cgroup weight, range is from 10 to 1000.
 	BlkioWeight uint16 `json:"blkio_weight"`
 
@@ -83,9 +97,6 @@ type Cgroup struct {
 	// Hugetlb limit (in bytes)
 	HugetlbLimit []*HugepageLimit `json:"hugetlb_limit"`
 
-	// Parent slice to use for systemd TODO: remove in favor or parent
-	Slice string `json:"slice"`
-
 	// Whether to disable OOM Killer
 	OomKillDisable bool `json:"oom_kill_disable"`
 

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/configs/cgroup_unsupported.go

@@ -0,0 +1,6 @@
+// +build !windows,!linux,!freebsd
+
+package configs
+
+type Cgroup struct {
+}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/configs/device_defaults.go

@@ -82,20 +82,6 @@ var (
 			Minor:       1,
 			Permissions: "rwm",
 		},
-		{
-			Path:        "/dev/tty0",
-			Type:        'c',
-			Major:       4,
-			Minor:       0,
-			Permissions: "rwm",
-		},
-		{
-			Path:        "/dev/tty1",
-			Type:        'c',
-			Major:       4,
-			Minor:       1,
-			Permissions: "rwm",
-		},
 		// /dev/pts/ - pts namespaces are "coming soon"
 		{
 			Path:        "",

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/console_freebsd.go

@@ -6,8 +6,8 @@ import (
 	"errors"
 )
 
-// newConsole returns an initalized console that can be used within a container by copying bytes
+// NewConsole returns an initalized console that can be used within a container by copying bytes
 // from the master side to the slave that is attached as the tty for the container's init process.
-func newConsole(uid, gid int) (Console, error) {
+func NewConsole(uid, gid int) (Console, error) {
 	return nil, errors.New("libcontainer console is not supported on FreeBSD")
 }

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/console_linux.go

@@ -10,9 +10,9 @@ import (
 	"github.com/opencontainers/runc/libcontainer/label"
 )
 
-// newConsole returns an initalized console that can be used within a container by copying bytes
+// NewConsole returns an initalized console that can be used within a container by copying bytes
 // from the master side to the slave that is attached as the tty for the container's init process.
-func newConsole(uid, gid int) (Console, error) {
+func NewConsole(uid, gid int) (Console, error) {
 	master, err := os.OpenFile("/dev/ptmx", syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0)
 	if err != nil {
 		return nil, err

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/console_windows.go

@@ -1,7 +1,7 @@
 package libcontainer
 
-// newConsole returns an initalized console that can be used within a container
+// NewConsole returns an initalized console that can be used within a container
-func newConsole(uid, gid int) (Console, error) {
+func NewConsole(uid, gid int) (Console, error) {
 	return &windowsConsole{}, nil
 }
 

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/container.go

@@ -30,6 +30,23 @@ const (
 	Destroyed
 )
 
+func (s Status) String() string {
+	switch s {
+	case Running:
+		return "running"
+	case Pausing:
+		return "pausing"
+	case Paused:
+		return "paused"
+	case Checkpointed:
+		return "checkpointed"
+	case Destroyed:
+		return "destroyed"
+	default:
+		return "undefined"
+	}
+}
+
 // BaseState represents the platform agnostic pieces relating to a
 // running container's state
 type BaseState struct {

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/container_linux.go

@@ -3,8 +3,10 @@
 package libcontainer
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -19,6 +21,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/opencontainers/runc/libcontainer/criurpc"
+	"github.com/vishvananda/netlink/nl"
 )
 
 const stdioFdCount = 3
@@ -34,6 +37,7 @@ type linuxContainer struct {
 	criuPath      string
 	m             sync.Mutex
 	criuVersion   int
+	state         containerState
 }
 
 // State represents a running container's state
@@ -100,6 +104,12 @@ type Container interface {
 	// errors:
 	// Systemerror - System error.
 	NotifyOOM() (<-chan struct{}, error)
+
+	// NotifyMemoryPressure returns a read-only channel signaling when the container reaches a given pressure level
+	//
+	// errors:
+	// Systemerror - System error.
+	NotifyMemoryPressure(level PressureLevel) (<-chan struct{}, error)
 }
 
 // ID returns the container's unique ID
@@ -125,7 +135,7 @@ func (c *linuxContainer) State() (*State, error) {
 }
 
 func (c *linuxContainer) Processes() ([]int, error) {
-	pids, err := c.cgroupManager.GetPids()
+	pids, err := c.cgroupManager.GetAllPids()
 	if err != nil {
 		return nil, newSystemError(err)
 	}
@@ -179,8 +189,12 @@ func (c *linuxContainer) Start(process *Process) error {
 		}
 		return newSystemError(err)
 	}
+	c.state = &runningState{
+		c: c,
+	}
 	if doInit {
-		c.updateState(parent)
+		if err := c.updateState(parent); err != nil {
+			return err
 		}
 		if c.config.Hooks != nil {
 			s := configs.HookState{
@@ -198,6 +212,7 @@ func (c *linuxContainer) Start(process *Process) error {
 				}
 			}
 		}
+	}
 	return nil
 }
 
@@ -218,7 +233,7 @@ func (c *linuxContainer) newParentProcess(p *Process, doInit bool) (parentProces
 		return nil, newSystemError(err)
 	}
 	if !doInit {
-		return c.newSetnsProcess(p, cmd, parentPipe, childPipe), nil
+		return c.newSetnsProcess(p, cmd, parentPipe, childPipe)
 	}
 	return c.newInitProcess(p, cmd, parentPipe, childPipe)
 }
@@ -247,7 +262,7 @@ func (c *linuxContainer) commandTemplate(p *Process, childPipe *os.File) (*exec.
 }
 
 func (c *linuxContainer) newInitProcess(p *Process, cmd *exec.Cmd, parentPipe, childPipe *os.File) (*initProcess, error) {
-	t := "_LIBCONTAINER_INITTYPE=standard"
+	t := "_LIBCONTAINER_INITTYPE=" + string(initStandard)
 	cloneFlags := c.config.Namespaces.CloneFlags()
 	if cloneFlags&syscall.CLONE_NEWUSER != 0 {
 		if err := c.addUidGidMappings(cmd.SysProcAttr); err != nil {
@@ -273,13 +288,13 @@ func (c *linuxContainer) newInitProcess(p *Process, cmd *exec.Cmd, parentPipe, c
 	}, nil
 }
 
-func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, parentPipe, childPipe *os.File) *setnsProcess {
+func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, parentPipe, childPipe *os.File) (*setnsProcess, error) {
-	cmd.Env = append(cmd.Env,
+	cmd.Env = append(cmd.Env, "_LIBCONTAINER_INITTYPE="+string(initSetns))
-		fmt.Sprintf("_LIBCONTAINER_INITPID=%d", c.initProcess.pid()),
+	// for setns process, we dont have to set cloneflags as the process namespaces
-		"_LIBCONTAINER_INITTYPE=setns",
+	// will only be set via setns syscall
-	)
+	data, err := c.bootstrapData(0, c.initProcess.pid(), p.consolePath)
-	if p.consolePath != "" {
+	if err != nil {
-		cmd.Env = append(cmd.Env, "_LIBCONTAINER_CONSOLE_PATH="+p.consolePath)
+		return nil, err
 	}
 	// TODO: set on container for process management
 	return &setnsProcess{
@@ -289,7 +304,8 @@ func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, parentPipe,
 		parentPipe:    parentPipe,
 		config:        c.newInitConfig(p),
 		process:       p,
-	}
+		bootstrapData: data,
+	}, nil
 }
 
 func (c *linuxContainer) newInitConfig(process *Process) *initConfig {
@@ -316,54 +332,53 @@ func newPipe() (parent *os.File, child *os.File, err error) {
 func (c *linuxContainer) Destroy() error {
 	c.m.Lock()
 	defer c.m.Unlock()
-	status, err := c.currentStatus()
+	return c.state.destroy()
-	if err != nil {
-		return err
-	}
-	if status != Destroyed {
-		return newGenericError(fmt.Errorf("container is not destroyed"), ContainerNotStopped)
-	}
-	if !c.config.Namespaces.Contains(configs.NEWPID) {
-		if err := killCgroupProcesses(c.cgroupManager); err != nil {
-			logrus.Warn(err)
-		}
-	}
-	err = c.cgroupManager.Destroy()
-	if rerr := os.RemoveAll(c.root); err == nil {
-		err = rerr
-	}
-	c.initProcess = nil
-	if c.config.Hooks != nil {
-		s := configs.HookState{
-			Version: c.config.Version,
-			ID:      c.id,
-			Root:    c.config.Rootfs,
-		}
-		for _, hook := range c.config.Hooks.Poststop {
-			if err := hook.Run(s); err != nil {
-				return err
-			}
-		}
-	}
-	return err
 }
 
 func (c *linuxContainer) Pause() error {
 	c.m.Lock()
 	defer c.m.Unlock()
-	return c.cgroupManager.Freeze(configs.Frozen)
+	status, err := c.currentStatus()
+	if err != nil {
+		return err
+	}
+	if status != Running {
+		return newGenericError(fmt.Errorf("container not running"), ContainerNotRunning)
+	}
+	if err := c.cgroupManager.Freeze(configs.Frozen); err != nil {
+		return err
+	}
+	return c.state.transition(&pausedState{
+		c: c,
+	})
 }
 
 func (c *linuxContainer) Resume() error {
 	c.m.Lock()
 	defer c.m.Unlock()
-	return c.cgroupManager.Freeze(configs.Thawed)
+	status, err := c.currentStatus()
+	if err != nil {
+		return err
+	}
+	if status != Paused {
+		return newGenericError(fmt.Errorf("container not paused"), ContainerNotPaused)
+	}
+	if err := c.cgroupManager.Freeze(configs.Thawed); err != nil {
+		return err
+	}
+	return c.state.transition(&runningState{
+		c: c,
+	})
 }
 
 func (c *linuxContainer) NotifyOOM() (<-chan struct{}, error) {
 	return notifyOnOOM(c.cgroupManager.GetPaths())
 }
 
+func (c *linuxContainer) NotifyMemoryPressure(level PressureLevel) (<-chan struct{}, error) {
+	return notifyMemoryPressure(c.cgroupManager.GetPaths(), level)
+}
+
 // XXX debug support, remove when debugging done.
 func addArgsFromEnv(evar string, args *[]string) {
 	if e := os.Getenv(evar); e != "" {
@@ -455,7 +470,7 @@ func (c *linuxContainer) Checkpoint(criuOpts *CriuOpts) error {
 	}
 
 	if criuOpts.ImagesDirectory == "" {
-		criuOpts.ImagesDirectory = filepath.Join(c.root, "criu.image")
+		return fmt.Errorf("invalid directory to save checkpoint")
 	}
 
 	// Since a container can be C/R'ed multiple times,
@@ -574,11 +589,9 @@ func (c *linuxContainer) addCriuRestoreMount(req *criurpc.CriuReq, m *configs.Mo
 func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error {
 	c.m.Lock()
 	defer c.m.Unlock()
-
 	if err := c.checkCriuVersion("1.5.2"); err != nil {
 		return err
 	}
-
 	if criuOpts.WorkDirectory == "" {
 		criuOpts.WorkDirectory = filepath.Join(c.root, "criu.work")
 	}
@@ -587,22 +600,19 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error {
 	if err := os.Mkdir(criuOpts.WorkDirectory, 0655); err != nil && !os.IsExist(err) {
 		return err
 	}
-
 	workDir, err := os.Open(criuOpts.WorkDirectory)
 	if err != nil {
 		return err
 	}
 	defer workDir.Close()
-
 	if criuOpts.ImagesDirectory == "" {
-		criuOpts.ImagesDirectory = filepath.Join(c.root, "criu.image")
+		return fmt.Errorf("invalid directory to restore checkpoint")
 	}
 	imageDir, err := os.Open(criuOpts.ImagesDirectory)
 	if err != nil {
 		return err
 	}
 	defer imageDir.Close()
-
 	// CRIU has a few requirements for a root directory:
 	// * it must be a mount point
 	// * its parent must not be overmounted
@@ -613,18 +623,15 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error {
 		return err
 	}
 	defer os.Remove(root)
-
 	root, err = filepath.EvalSymlinks(root)
 	if err != nil {
 		return err
 	}
-
 	err = syscall.Mount(c.config.Rootfs, root, "", syscall.MS_BIND|syscall.MS_REC, "")
 	if err != nil {
 		return err
 	}
 	defer syscall.Unmount(root, syscall.MNT_DETACH)
-
 	t := criurpc.CriuReqType_RESTORE
 	req := &criurpc.CriuReq{
 		Type: &t,
@@ -692,15 +699,13 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error {
 		fds    []string
 		fdJSON []byte
 	)
-
 	if fdJSON, err = ioutil.ReadFile(filepath.Join(criuOpts.ImagesDirectory, descriptorsFilename)); err != nil {
 		return err
 	}
 
-	if err = json.Unmarshal(fdJSON, &fds); err != nil {
+	if err := json.Unmarshal(fdJSON, &fds); err != nil {
 		return err
 	}
-
 	for i := range fds {
 		if s := fds[i]; strings.Contains(s, "pipe:") {
 			inheritFd := new(criurpc.InheritFd)
@@ -709,12 +714,7 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error {
 			req.Opts.InheritFd = append(req.Opts.InheritFd, inheritFd)
 		}
 	}
-
+	return c.criuSwrk(process, req, criuOpts, true)
-	err = c.criuSwrk(process, req, criuOpts, true)
-	if err != nil {
-		return err
-	}
-	return nil
 }
 
 func (c *linuxContainer) criuApplyCgroups(pid int, req *criurpc.CriuReq) error {
@@ -909,46 +909,43 @@ func (c *linuxContainer) criuNotifications(resp *criurpc.CriuResp, process *Proc
 	if notify == nil {
 		return fmt.Errorf("invalid response: %s", resp.String())
 	}
-
 	switch {
 	case notify.GetScript() == "post-dump":
-		if !opts.LeaveRunning {
 		f, err := os.Create(filepath.Join(c.root, "checkpoint"))
 		if err != nil {
 			return err
 		}
 		f.Close()
-		}
-		break
-
 	case notify.GetScript() == "network-unlock":
 		if err := unlockNetwork(c.config); err != nil {
 			return err
 		}
-		break
-
 	case notify.GetScript() == "network-lock":
 		if err := lockNetwork(c.config); err != nil {
 			return err
 		}
-		break
-
 	case notify.GetScript() == "post-restore":
 		pid := notify.GetPid()
 		r, err := newRestoredProcess(int(pid), fds)
 		if err != nil {
 			return err
 		}
-
+		process.ops = r
-		// TODO: crosbymichael restore previous process information by saving the init process information in
+		if err := c.state.transition(&restoredState{
-		// the container's state file or separate process state files.
+			imageDir: opts.ImagesDirectory,
+			c:        c,
+		}); err != nil {
+			return err
+		}
 		if err := c.updateState(r); err != nil {
 			return err
 		}
-		process.ops = r
+		if err := os.Remove(filepath.Join(c.root, "checkpoint")); err != nil {
-		break
+			if !os.IsNotExist(err) {
+				logrus.Error(err)
+			}
+		}
 	}
-
 	return nil
 }
 
@@ -958,66 +955,130 @@ func (c *linuxContainer) updateState(process parentProcess) error {
 	if err != nil {
 		return err
 	}
+	return c.saveState(state)
+}
+
+func (c *linuxContainer) saveState(s *State) error {
 	f, err := os.Create(filepath.Join(c.root, stateFilename))
 	if err != nil {
 		return err
 	}
 	defer f.Close()
-	os.Remove(filepath.Join(c.root, "checkpoint"))
+	return json.NewEncoder(f).Encode(s)
-	return json.NewEncoder(f).Encode(state)
+}
+
+func (c *linuxContainer) deleteState() error {
+	return os.Remove(filepath.Join(c.root, stateFilename))
 }
 
 func (c *linuxContainer) currentStatus() (Status, error) {
-	if _, err := os.Stat(filepath.Join(c.root, "checkpoint")); err == nil {
+	if err := c.refreshState(); err != nil {
-		return Checkpointed, nil
+		return -1, err
 	}
+	return c.state.status(), nil
+}
+
+// refreshState needs to be called to verify that the current state on the
+// container is what is true.  Because consumers of libcontainer can use it
+// out of process we need to verify the container's status based on runtime
+// information and not rely on our in process info.
+func (c *linuxContainer) refreshState() error {
+	paused, err := c.isPaused()
+	if err != nil {
+		return err
+	}
+	if paused {
+		return c.state.transition(&pausedState{c: c})
+	}
+	running, err := c.isRunning()
+	if err != nil {
+		return err
+	}
+	if running {
+		return c.state.transition(&runningState{c: c})
+	}
+	return c.state.transition(&stoppedState{c: c})
+}
+
+func (c *linuxContainer) isRunning() (bool, error) {
 	if c.initProcess == nil {
-		return Destroyed, nil
+		return false, nil
 	}
 	// return Running if the init process is alive
 	if err := syscall.Kill(c.initProcess.pid(), 0); err != nil {
 		if err == syscall.ESRCH {
-			return Destroyed, nil
+			return false, nil
 		}
-		return 0, newSystemError(err)
+		return false, newSystemError(err)
 	}
-	if c.config.Cgroups != nil && c.config.Cgroups.Freezer == configs.Frozen {
+	return true, nil
-		return Paused, nil
+}
+
+func (c *linuxContainer) isPaused() (bool, error) {
+	data, err := ioutil.ReadFile(filepath.Join(c.cgroupManager.GetPaths()["freezer"], "freezer.state"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
 		}
-	return Running, nil
+		return false, newSystemError(err)
+	}
+	return bytes.Equal(bytes.TrimSpace(data), []byte("FROZEN")), nil
 }
 
 func (c *linuxContainer) currentState() (*State, error) {
-	status, err := c.currentStatus()
+	var (
-	if err != nil {
+		startTime           string
-		return nil, err
+		externalDescriptors []string
-	}
+		pid                 = -1
-	if status == Destroyed {
+	)
-		return nil, newGenericError(fmt.Errorf("container destroyed"), ContainerNotExists)
+	if c.initProcess != nil {
-	}
+		pid = c.initProcess.pid()
-	startTime, err := c.initProcess.startTime()
+		startTime, _ = c.initProcess.startTime()
-	if err != nil {
+		externalDescriptors = c.initProcess.externalDescriptors()
-		return nil, newSystemError(err)
 	}
 	state := &State{
 		BaseState: BaseState{
 			ID:                   c.ID(),
 			Config:               *c.config,
-			InitProcessPid:       c.initProcess.pid(),
+			InitProcessPid:       pid,
 			InitProcessStartTime: startTime,
 		},
 		CgroupPaths:         c.cgroupManager.GetPaths(),
 		NamespacePaths:      make(map[configs.NamespaceType]string),
-		ExternalDescriptors: c.initProcess.externalDescriptors(),
+		ExternalDescriptors: externalDescriptors,
 	}
+	if pid > 0 {
 		for _, ns := range c.config.Namespaces {
-		state.NamespacePaths[ns.Type] = ns.GetPath(c.initProcess.pid())
+			state.NamespacePaths[ns.Type] = ns.GetPath(pid)
 		}
 		for _, nsType := range configs.NamespaceTypes() {
 			if _, ok := state.NamespacePaths[nsType]; !ok {
 				ns := configs.Namespace{Type: nsType}
-			state.NamespacePaths[ns.Type] = ns.GetPath(c.initProcess.pid())
+				state.NamespacePaths[ns.Type] = ns.GetPath(pid)
+			}
 		}
 	}
 	return state, nil
 }
+
+// bootstrapData encodes the necessary data in netlink binary format as a io.Reader.
+// Consumer can write the data to a bootstrap program such as one that uses
+// nsenter package to bootstrap the container's init process correctly, i.e. with
+// correct namespaces, uid/gid mapping etc.
+func (c *linuxContainer) bootstrapData(cloneFlags uintptr, pid int, consolePath string) (io.Reader, error) {
+	// create the netlink message
+	r := nl.NewNetlinkRequest(int(InitMsg), 0)
+	// write pid
+	r.AddData(&Int32msg{
+		Type:  PidAttr,
+		Value: uint32(pid),
+	})
+	// write console path
+	if consolePath != "" {
+		r.AddData(&Bytemsg{
+			Type:  ConsolePathAttr,
+			Value: []byte(consolePath),
+		})
+	}
+	return bytes.NewReader(r.Serialize()), nil
+}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/error.go

@@ -16,12 +16,14 @@ const (
 	ContainerPaused
 	ContainerNotStopped
 	ContainerNotRunning
+	ContainerNotPaused
 
 	// Process errors
 	ProcessNotExecuted
 
 	// Common errors
 	ConfigInvalid
+	ConsoleExists
 	SystemError
 )
 
@@ -43,6 +45,10 @@ func (c ErrorCode) String() string {
 		return "Container is not stopped"
 	case ContainerNotRunning:
 		return "Container is not running"
+	case ConsoleExists:
+		return "Console exists for process"
+	case ContainerNotPaused:
+		return "Container is not paused"
 	default:
 		return "Unknown error"
 	}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/factory_linux.go

@@ -5,7 +5,6 @@ package libcontainer
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -166,7 +165,7 @@ func (l *LinuxFactory) Create(id string, config *configs.Config) (Container, err
 	if err := os.MkdirAll(containerRoot, 0700); err != nil {
 		return nil, newGenericError(err, SystemError)
 	}
-	return &linuxContainer{
+	c := &linuxContainer{
 		id:            id,
 		root:          containerRoot,
 		config:        config,
@@ -174,7 +173,9 @@ func (l *LinuxFactory) Create(id string, config *configs.Config) (Container, err
 		initArgs:      l.InitArgs,
 		criuPath:      l.CriuPath,
 		cgroupManager: l.NewCgroupsManager(config.Cgroups, nil),
-	}, nil
+	}
+	c.state = &stoppedState{c: c}
+	return c, nil
 }
 
 func (l *LinuxFactory) Load(id string) (Container, error) {
@@ -191,7 +192,7 @@ func (l *LinuxFactory) Load(id string) (Container, error) {
 		processStartTime: state.InitProcessStartTime,
 		fds:              state.ExternalDescriptors,
 	}
-	return &linuxContainer{
+	c := &linuxContainer{
 		initProcess:   r,
 		id:            id,
 		config:        &state.Config,
@@ -200,7 +201,12 @@ func (l *LinuxFactory) Load(id string) (Container, error) {
 		criuPath:      l.CriuPath,
 		cgroupManager: l.NewCgroupsManager(state.Config.Cgroups, state.CgroupPaths),
 		root:          containerRoot,
-	}, nil
+	}
+	c.state = &nullState{c: c}
+	if err := c.refreshState(); err != nil {
+		return nil, err
+	}
+	return c, nil
 }
 
 func (l *LinuxFactory) Type() string {
@@ -222,21 +228,29 @@ func (l *LinuxFactory) StartInitialization() (err error) {
 	// clear the current process's environment to clean any libcontainer
 	// specific env vars.
 	os.Clearenv()
+	var i initer
 	defer func() {
 		// if we have an error during the initialization of the container's init then send it back to the
 		// parent process in the form of an initError.
 		if err != nil {
-			// ensure that any data sent from the parent is consumed so it doesn't
+			if _, ok := i.(*linuxStandardInit); ok {
-			// receive ECONNRESET when the child writes to the pipe.
+				//  Synchronisation only necessary for standard init.
-			ioutil.ReadAll(pipe)
+				if err := json.NewEncoder(pipe).Encode(procError); err != nil {
+					panic(err)
+				}
+			}
 			if err := json.NewEncoder(pipe).Encode(newSystemError(err)); err != nil {
 				panic(err)
 			}
+		} else {
+			if err := json.NewEncoder(pipe).Encode(procStart); err != nil {
+				panic(err)
+			}
 		}
 		// ensure that this pipe is always closed
 		pipe.Close()
 	}()
-	i, err := newContainerInit(it, pipe)
+	i, err = newContainerInit(it, pipe)
 	if err != nil {
 		return err
 	}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/generic_error.go

@@ -9,6 +9,15 @@ import (
 	"github.com/opencontainers/runc/libcontainer/stacktrace"
 )
 
+type syncType uint8
+
+const (
+	procReady syncType = iota
+	procError
+	procStart
+	procRun
+)
+
 var errorTemplate = template.Must(template.New("error").Parse(`Timestamp: {{.Timestamp}}
 Code: {{.ECode}}
 {{if .Message }}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/hack/validate.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# This script runs all validations
-
-validate() {
-    export MAKEDIR=/go/src/github.com/docker/docker/hack/make
-    sed -i 's!docker/docker!opencontainers/runc/libcontainer!' /go/src/github.com/docker/docker/hack/make/.validate
-    bash /go/src/github.com/docker/docker/hack/make/validate-dco
-    bash /go/src/github.com/docker/docker/hack/make/validate-gofmt
-    go get golang.org/x/tools/cmd/vet
-    bash /go/src/github.com/docker/docker/hack/make/validate-vet
-}
-
-# run validations
-validate

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/init_linux.go

@@ -5,6 +5,7 @@ package libcontainer
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"os"
@@ -73,6 +74,7 @@ func newContainerInit(t initType, pipe *os.File) (initer, error) {
 		}, nil
 	case initStandard:
 		return &linuxStandardInit{
+			pipe:      pipe,
 			parentPid: syscall.Getppid(),
 			config:    config,
 		}, nil
@@ -140,6 +142,28 @@ func finalizeNamespace(config *initConfig) error {
 	return nil
 }
 
+// syncParentReady sends to the given pipe a JSON payload which indicates that
+// the init is ready to Exec the child process. It then waits for the parent to
+// indicate that it is cleared to Exec.
+func syncParentReady(pipe io.ReadWriter) error {
+	// Tell parent.
+	if err := json.NewEncoder(pipe).Encode(procReady); err != nil {
+		return err
+	}
+
+	// Wait for parent to give the all-clear.
+	var procSync syncType
+	if err := json.NewDecoder(pipe).Decode(&procSync); err != nil {
+		if err == io.EOF {
+			return fmt.Errorf("parent closed synchronisation channel")
+		}
+		if procSync != procRun {
+			return fmt.Errorf("invalid synchronisation flag from parent")
+		}
+	}
+	return nil
+}
+
 // joinExistingNamespaces gets all the namespace paths specified for the container and
 // does a setns on the namespace fd so that the current process joins the namespace.
 func joinExistingNamespaces(namespaces []configs.Namespace) error {
@@ -309,7 +333,7 @@ func killCgroupProcesses(m cgroups.Manager) error {
 	if err := m.Freeze(configs.Frozen); err != nil {
 		logrus.Warn(err)
 	}
-	pids, err := m.GetPids()
+	pids, err := m.GetAllPids()
 	if err != nil {
 		m.Freeze(configs.Thawed)
 		return err

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/message_linux.go

@@ -0,0 +1,62 @@
+// +build linux
+
+package libcontainer
+
+import (
+	"syscall"
+
+	"github.com/vishvananda/netlink/nl"
+)
+
+// list of known message types we want to send to bootstrap program
+// The number is randomly chosen to not conflict with known netlink types
+const (
+	InitMsg         uint16 = 62000
+	PidAttr         uint16 = 27281
+	ConsolePathAttr uint16 = 27282
+	// When syscall.NLA_HDRLEN is in gccgo, take this out.
+	syscall_NLA_HDRLEN = (syscall.SizeofNlAttr + syscall.NLA_ALIGNTO - 1) & ^(syscall.NLA_ALIGNTO - 1)
+)
+
+type Int32msg struct {
+	Type  uint16
+	Value uint32
+}
+
+// int32msg has the following representation
+// | nlattr len | nlattr type |
+// | uint32 value             |
+func (msg *Int32msg) Serialize() []byte {
+	buf := make([]byte, msg.Len())
+	native := nl.NativeEndian()
+	native.PutUint16(buf[0:2], uint16(msg.Len()))
+	native.PutUint16(buf[2:4], msg.Type)
+	native.PutUint32(buf[4:8], msg.Value)
+	return buf
+}
+
+func (msg *Int32msg) Len() int {
+	return syscall_NLA_HDRLEN + 4
+}
+
+// bytemsg has the following representation
+// | nlattr len | nlattr type |
+// | value              | pad |
+type Bytemsg struct {
+	Type  uint16
+	Value []byte
+}
+
+func (msg *Bytemsg) Serialize() []byte {
+	l := msg.Len()
+	buf := make([]byte, (l+syscall.NLA_ALIGNTO-1) & ^(syscall.NLA_ALIGNTO-1))
+	native := nl.NativeEndian()
+	native.PutUint16(buf[0:2], uint16(l))
+	native.PutUint16(buf[2:4], msg.Type)
+	copy(buf[4:], msg.Value)
+	return buf
+}
+
+func (msg *Bytemsg) Len() int {
+	return syscall_NLA_HDRLEN + len(msg.Value) + 1 // null-terminated
+}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/network_linux.go

@@ -93,7 +93,7 @@ func (l *loopback) create(n *network, nspid int) error {
 }
 
 func (l *loopback) initialize(config *network) error {
-	return netlink.LinkSetUp(&netlink.Device{netlink.LinkAttrs{Name: "lo"}})
+	return netlink.LinkSetUp(&netlink.Device{LinkAttrs: netlink.LinkAttrs{Name: "lo"}})
 }
 
 func (l *loopback) attach(n *configs.Network) (err error) {
@@ -111,7 +111,7 @@ type veth struct {
 }
 
 func (v *veth) detach(n *configs.Network) (err error) {
-	return netlink.LinkSetMaster(&netlink.Device{netlink.LinkAttrs{Name: n.HostInterfaceName}}, nil)
+	return netlink.LinkSetMaster(&netlink.Device{LinkAttrs: netlink.LinkAttrs{Name: n.HostInterfaceName}}, nil)
 }
 
 // attach a container network interface to an external network

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/notify_linux.go

@@ -12,31 +12,32 @@ import (
 
 const oomCgroupName = "memory"
 
-// notifyOnOOM returns channel on which you can expect event about OOM,
+type PressureLevel uint
-// if process died without OOM this channel will be closed.
+
-// s is current *libcontainer.State for container.
+const (
-func notifyOnOOM(paths map[string]string) (<-chan struct{}, error) {
+	LowPressure PressureLevel = iota
-	dir := paths[oomCgroupName]
+	MediumPressure
-	if dir == "" {
+	CriticalPressure
-		return nil, fmt.Errorf("There is no path for %q in state", oomCgroupName)
+)
-	}
+
-	oomControl, err := os.Open(filepath.Join(dir, "memory.oom_control"))
+func registerMemoryEvent(cgDir string, evName string, arg string) (<-chan struct{}, error) {
+	evFile, err := os.Open(filepath.Join(cgDir, evName))
 	if err != nil {
 		return nil, err
 	}
 	fd, _, syserr := syscall.RawSyscall(syscall.SYS_EVENTFD2, 0, syscall.FD_CLOEXEC, 0)
 	if syserr != 0 {
-		oomControl.Close()
+		evFile.Close()
 		return nil, syserr
 	}
 
 	eventfd := os.NewFile(fd, "eventfd")
 
-	eventControlPath := filepath.Join(dir, "cgroup.event_control")
+	eventControlPath := filepath.Join(cgDir, "cgroup.event_control")
-	data := fmt.Sprintf("%d %d", eventfd.Fd(), oomControl.Fd())
+	data := fmt.Sprintf("%d %d %s", eventfd.Fd(), evFile.Fd(), arg)
 	if err := ioutil.WriteFile(eventControlPath, []byte(data), 0700); err != nil {
 		eventfd.Close()
-		oomControl.Close()
+		evFile.Close()
 		return nil, err
 	}
 	ch := make(chan struct{})
@@ -44,7 +45,7 @@ func notifyOnOOM(paths map[string]string) (<-chan struct{}, error) {
 		defer func() {
 			close(ch)
 			eventfd.Close()
-			oomControl.Close()
+			evFile.Close()
 		}()
 		buf := make([]byte, 8)
 		for {
@@ -61,3 +62,28 @@ func notifyOnOOM(paths map[string]string) (<-chan struct{}, error) {
 	}()
 	return ch, nil
 }
+
+// notifyOnOOM returns channel on which you can expect event about OOM,
+// if process died without OOM this channel will be closed.
+func notifyOnOOM(paths map[string]string) (<-chan struct{}, error) {
+	dir := paths[oomCgroupName]
+	if dir == "" {
+		return nil, fmt.Errorf("path %q missing", oomCgroupName)
+	}
+
+	return registerMemoryEvent(dir, "memory.oom_control", "")
+}
+
+func notifyMemoryPressure(paths map[string]string, level PressureLevel) (<-chan struct{}, error) {
+	dir := paths[oomCgroupName]
+	if dir == "" {
+		return nil, fmt.Errorf("path %q missing", oomCgroupName)
+	}
+
+	if level > CriticalPressure {
+		return nil, fmt.Errorf("invalid pressure level %d", level)
+	}
+
+	levelStr := []string{"low", "medium", "critical"}[level]
+	return registerMemoryEvent(dir, "memory.pressure_level", levelStr)
+}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c

@@ -17,6 +17,11 @@
 #include <sched.h>
 #include <signal.h>
 
+#include <linux/netlink.h>
+#include <linux/types.h>
+#include <stdint.h>
+#include <sys/socket.h>
+
 /* All arguments should be above stack, because it grows down */
 struct clone_arg {
 	/*
@@ -63,24 +68,33 @@ static int clone_parent(jmp_buf * env)
 	return child;
 }
 
+static uint32_t readint32(char *buf)
+{
+	return *(uint32_t *) buf;
+}
+
+// list of known message types we want to send to bootstrap program
+// These are defined in libcontainer/message_linux.go
+#define INIT_MSG 62000
+#define PID_ATTR 27281
+#define CONSOLE_PATH_ATTR 27282
+
 void nsexec()
 {
 	char *namespaces[] = { "ipc", "uts", "net", "pid", "mnt", "user" };
 	const int num = sizeof(namespaces) / sizeof(char *);
 	jmp_buf env;
 	char buf[PATH_MAX], *val;
-	int i, tfd, self_tfd, child, len, pipenum, consolefd = -1;
+	int i, tfd, self_tfd, child, n, len, pipenum, consolefd = -1;
-	pid_t pid;
+	pid_t pid = 0;
-	char *console;
 
-	val = getenv("_LIBCONTAINER_INITPID");
+	// if we dont have INITTYPE or this is the init process, skip the bootstrap process
-	if (val == NULL)
+	val = getenv("_LIBCONTAINER_INITTYPE");
+	if (val == NULL || strcmp(val, "standard") == 0) {
 		return;
-
+	}
-	pid = atoi(val);
+	if (strcmp(val, "setns") != 0) {
-	snprintf(buf, sizeof(buf), "%d", pid);
+		pr_perror("Invalid inittype %s", val);
-	if (strcmp(val, buf)) {
-		pr_perror("Unable to parse _LIBCONTAINER_INITPID");
 		exit(1);
 	}
 
@@ -89,7 +103,6 @@ void nsexec()
 		pr_perror("Child pipe not found");
 		exit(1);
 	}
-
 	pipenum = atoi(val);
 	snprintf(buf, sizeof(buf), "%d", pipenum);
 	if (strcmp(val, buf)) {
@@ -97,13 +110,56 @@ void nsexec()
 		exit(1);
 	}
 
-	console = getenv("_LIBCONTAINER_CONSOLE_PATH");
+	char nlbuf[NLMSG_HDRLEN];
-	if (console != NULL) {
+	struct nlmsghdr *nh;
-		consolefd = open(console, O_RDWR);
+	if ((n = read(pipenum, nlbuf, NLMSG_HDRLEN)) != NLMSG_HDRLEN) {
-		if (consolefd < 0) {
+		pr_perror("Failed to read netlink header, got %d", n);
-			pr_perror("Failed to open console %s", console);
 		exit(1);
 	}
+
+	nh = (struct nlmsghdr *)nlbuf;
+	if (nh->nlmsg_type == NLMSG_ERROR) {
+		pr_perror("Invalid netlink header message");
+		exit(1);
+	}
+	if (nh->nlmsg_type != INIT_MSG) {
+		pr_perror("Unexpected netlink message type %d", nh->nlmsg_type);
+		exit(1);
+	}
+	// read the netlink payload
+	len = NLMSG_PAYLOAD(nh, 0);
+	char data[len];
+	if ((n = read(pipenum, data, len)) != len) {
+		pr_perror("Failed to read netlink payload, got %d", n);
+		exit(1);
+	}
+
+	int start = 0;
+	struct nlattr *attr;
+	while (start < len) {
+		int payload_len;
+		attr = (struct nlattr *)((void *)data + start);
+		start += NLA_HDRLEN;
+		payload_len = attr->nla_len - NLA_HDRLEN;
+		switch (attr->nla_type) {
+		case PID_ATTR:
+			pid = (pid_t) readint32(data + start);
+			break;
+		case CONSOLE_PATH_ATTR:
+			consolefd = open((char *)data + start, O_RDWR);
+			if (consolefd < 0) {
+				pr_perror("Failed to open console %s", (char *)data + start);
+				exit(1);
+			}
+			break;
+		}
+		start += NLA_ALIGN(payload_len);
+	}
+
+	// required pid to be passed
+	if (pid == 0) {
+		pr_perror("missing pid");
+		exit(1);
 	}
 
 	/* Check that the specified process exists */
@@ -133,15 +189,13 @@ void nsexec()
 		}
 
 		/* Skip namespaces we're already part of */
-		if (fstatat(self_tfd, namespaces[i], &self_st, 0) != -1 &&
+		if (fstatat(self_tfd, namespaces[i], &self_st, 0) != -1 && st.st_ino == self_st.st_ino) {
-		    st.st_ino == self_st.st_ino) {
 			continue;
 		}
 
 		fd = openat(tfd, namespaces[i], O_RDONLY);
 		if (fd == -1) {
-			pr_perror("Failed to open ns file %s for ns %s", buf,
+			pr_perror("Failed to open ns file %s for ns %s", buf, namespaces[i]);
-				  namespaces[i]);
 			exit(1);
 		}
 		// Set the namespace.

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/process.go

@@ -78,12 +78,28 @@ func (p Process) Signal(sig os.Signal) error {
 	return p.ops.signal(sig)
 }
 
+// IO holds the process's STDIO
+type IO struct {
+	Stdin  io.WriteCloser
+	Stdout io.ReadCloser
+	Stderr io.ReadCloser
+}
+
 // NewConsole creates new console for process and returns it
 func (p *Process) NewConsole(rootuid int) (Console, error) {
-	console, err := newConsole(rootuid, rootuid)
+	console, err := NewConsole(rootuid, rootuid)
 	if err != nil {
 		return nil, err
 	}
 	p.consolePath = console.Path()
 	return console, nil
 }
+
+// ConsoleFromPath sets the process's console with the path provided
+func (p *Process) ConsoleFromPath(path string) error {
+	if p.consolePath != "" {
+		return newGenericError(fmt.Errorf("console path already exists for process"), ConsoleExists)
+	}
+	p.consolePath = path
+	return nil
+}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/process_linux.go

@@ -5,6 +5,7 @@ package libcontainer
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
@@ -48,6 +49,7 @@ type setnsProcess struct {
 	config        *initConfig
 	fds           []string
 	process       *Process
+	bootstrapData io.Reader
 }
 
 func (p *setnsProcess) startTime() (string, error) {
@@ -64,6 +66,16 @@ func (p *setnsProcess) signal(sig os.Signal) error {
 
 func (p *setnsProcess) start() (err error) {
 	defer p.parentPipe.Close()
+	err = p.cmd.Start()
+	p.childPipe.Close()
+	if err != nil {
+		return newSystemError(err)
+	}
+	if p.bootstrapData != nil {
+		if _, err := io.Copy(p.parentPipe, p.bootstrapData); err != nil {
+			return newSystemError(err)
+		}
+	}
 	if err = p.execSetns(); err != nil {
 		return newSystemError(err)
 	}
@@ -75,6 +87,7 @@ func (p *setnsProcess) start() (err error) {
 	if err := json.NewEncoder(p.parentPipe).Encode(p.config); err != nil {
 		return newSystemError(err)
 	}
+
 	if err := syscall.Shutdown(int(p.parentPipe.Fd()), syscall.SHUT_WR); err != nil {
 		return newSystemError(err)
 	}
@@ -84,6 +97,7 @@ func (p *setnsProcess) start() (err error) {
 	if err := json.NewDecoder(p.parentPipe).Decode(&ierr); err != nil && err != io.EOF {
 		return newSystemError(err)
 	}
+	// Must be done after Shutdown so the child will exit and we can wait for it.
 	if ierr != nil {
 		p.wait()
 		return newSystemError(ierr)
@@ -96,11 +110,6 @@ func (p *setnsProcess) start() (err error) {
 // before the go runtime boots, we wait on the process to die and receive the child's pid
 // over the provided pipe.
 func (p *setnsProcess) execSetns() error {
-	err := p.cmd.Start()
-	p.childPipe.Close()
-	if err != nil {
-		return newSystemError(err)
-	}
 	status, err := p.cmd.Process.Wait()
 	if err != nil {
 		p.cmd.Wait()
@@ -192,7 +201,6 @@ func (p *initProcess) start() (err error) {
 		return newSystemError(err)
 	}
 	p.setExternalDescriptors(fds)
-
 	// Do this before syncing with child so that no children
 	// can escape the cgroup
 	if err := p.manager.Apply(p.pid()); err != nil {
@@ -223,13 +231,58 @@ func (p *initProcess) start() (err error) {
 	if err := p.sendConfig(); err != nil {
 		return newSystemError(err)
 	}
+
+	var (
+		procSync syncType
+		sentRun  bool
+		ierr     *genericError
+	)
+
+loop:
+	for {
+		if err := json.NewDecoder(p.parentPipe).Decode(&procSync); err != nil {
+			if err == io.EOF {
+				break loop
+			}
+			return newSystemError(err)
+		}
+
+		switch procSync {
+		case procStart:
+			break loop
+		case procReady:
+			if err := p.manager.Set(p.config.Config); err != nil {
+				return newSystemError(err)
+			}
+			// Sync with child.
+			if err := json.NewEncoder(p.parentPipe).Encode(procRun); err != nil {
+				return newSystemError(err)
+			}
+			sentRun = true
+		case procError:
 			// wait for the child process to fully complete and receive an error message
 			// if one was encoutered
-	var ierr *genericError
 			if err := json.NewDecoder(p.parentPipe).Decode(&ierr); err != nil && err != io.EOF {
 				return newSystemError(err)
 			}
 			if ierr != nil {
+				break loop
+			}
+			// Programmer error.
+			panic("No error following JSON procError payload.")
+		default:
+			return newSystemError(fmt.Errorf("invalid JSON synchronisation payload from child"))
+		}
+	}
+	if !sentRun {
+		return newSystemError(fmt.Errorf("could not synchronise with container process"))
+	}
+	if err := syscall.Shutdown(int(p.parentPipe.Fd()), syscall.SHUT_WR); err != nil {
+		return newSystemError(err)
+	}
+	// Must be done after Shutdown so the child will exit and we can wait for it.
+	if ierr != nil {
+		p.wait()
 		return newSystemError(ierr)
 	}
 	return nil
@@ -267,8 +320,7 @@ func (p *initProcess) sendConfig() error {
 	if err := json.NewEncoder(p.parentPipe).Encode(p.config); err != nil {
 		return err
 	}
-	// shutdown writes for the parent side of the pipe
+	return nil
-	return syscall.Shutdown(int(p.parentPipe.Fd()), syscall.SHUT_WR)
 }
 
 func (p *initProcess) createNetworkInterfaces() error {
@@ -314,3 +366,44 @@ func getPipeFds(pid int) ([]string, error) {
 	}
 	return fds, nil
 }
+
+// InitializeIO creates pipes for use with the process's STDIO
+// and returns the opposite side for each
+func (p *Process) InitializeIO(rootuid int) (i *IO, err error) {
+	var fds []uintptr
+	i = &IO{}
+	// cleanup in case of an error
+	defer func() {
+		if err != nil {
+			for _, fd := range fds {
+				syscall.Close(int(fd))
+			}
+		}
+	}()
+	// STDIN
+	r, w, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+	fds = append(fds, r.Fd(), w.Fd())
+	p.Stdin, i.Stdin = r, w
+	// STDOUT
+	if r, w, err = os.Pipe(); err != nil {
+		return nil, err
+	}
+	fds = append(fds, r.Fd(), w.Fd())
+	p.Stdout, i.Stdout = w, r
+	// STDERR
+	if r, w, err = os.Pipe(); err != nil {
+		return nil, err
+	}
+	fds = append(fds, r.Fd(), w.Fd())
+	p.Stderr, i.Stderr = w, r
+	// change ownership of the pipes incase we are in a user namespace
+	for _, fd := range fds {
+		if err := syscall.Fchown(int(fd), rootuid, rootuid); err != nil {
+			return nil, err
+		}
+	}
+	return i, nil
+}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/rootfs_linux.go

@@ -18,6 +18,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/opencontainers/runc/libcontainer/label"
+	"github.com/opencontainers/runc/libcontainer/system"
 )
 
 const defaultMountFlags = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV
@@ -299,6 +300,24 @@ func checkMountDestination(rootfs, dest string) error {
 	invalidDestinations := []string{
 		"/proc",
 	}
+	// White list, it should be sub directories of invalid destinations
+	validDestinations := []string{
+		// These entries can be bind mounted by files emulated by fuse,
+		// so commands like top, free displays stats in container.
+		"/proc/cpuinfo",
+		"/proc/diskstats",
+		"/proc/meminfo",
+		"/proc/stats",
+	}
+	for _, valid := range validDestinations {
+		path, err := filepath.Rel(filepath.Join(rootfs, valid), dest)
+		if err != nil {
+			return err
+		}
+		if path == "." {
+			return nil
+		}
+	}
 	for _, invalid := range invalidDestinations {
 		path, err := filepath.Rel(filepath.Join(rootfs, invalid), dest)
 		if err != nil {
@@ -365,11 +384,12 @@ func reOpenDevNull() error {
 
 // Create the device nodes in the container.
 func createDevices(config *configs.Config) error {
+	useBindMount := system.RunningInUserNS() || config.Namespaces.Contains(configs.NEWUSER)
 	oldMask := syscall.Umask(0000)
 	for _, node := range config.Devices {
 		// containers running in a user namespace are not allowed to mknod
 		// devices so we can just bind mount it from the host.
-		if err := createDeviceNode(config.Rootfs, node, config.Namespaces.Contains(configs.NEWUSER)); err != nil {
+		if err := createDeviceNode(config.Rootfs, node, useBindMount); err != nil {
 			syscall.Umask(oldMask)
 			return err
 		}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/seccomp/fixtures/proc_self_status

@@ -0,0 +1,47 @@
+Name:   cat
+State:  R (running)
+Tgid:   19383
+Ngid:   0
+Pid:    19383
+PPid:   19275
+TracerPid:  0
+Uid:    1000    1000    1000    1000
+Gid:    1000    1000    1000    1000
+FDSize: 256
+Groups: 24 25 27 29 30 44 46 102 104 108 111 1000 1001
+NStgid: 19383
+NSpid:  19383
+NSpgid: 19383
+NSsid:  19275
+VmPeak:     5944 kB
+VmSize:     5944 kB
+VmLck:         0 kB
+VmPin:         0 kB
+VmHWM:       744 kB
+VmRSS:       744 kB
+VmData:      324 kB
+VmStk:       136 kB
+VmExe:        48 kB
+VmLib:      1776 kB
+VmPTE:        32 kB
+VmPMD:        12 kB
+VmSwap:        0 kB
+Threads:    1
+SigQ:   0/30067
+SigPnd: 0000000000000000
+ShdPnd: 0000000000000000
+SigBlk: 0000000000000000
+SigIgn: 0000000000000080
+SigCgt: 0000000000000000
+CapInh: 0000000000000000
+CapPrm: 0000000000000000
+CapEff: 0000000000000000
+CapBnd: 0000003fffffffff
+CapAmb: 0000000000000000
+Seccomp:    0
+Cpus_allowed:   f
+Cpus_allowed_list:  0-3
+Mems_allowed:   00000000,00000001
+Mems_allowed_list:  0
+voluntary_ctxt_switches:    0
+nonvoluntary_ctxt_switches: 1

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/seccomp/seccomp_linux.go

@@ -3,8 +3,11 @@
 package seccomp
 
 import (
+	"bufio"
 	"fmt"
 	"log"
+	"os"
+	"strings"
 	"syscall"
 
 	"github.com/opencontainers/runc/libcontainer/configs"
@@ -17,6 +20,9 @@ var (
 	actKill  = libseccomp.ActKill
 	actTrace = libseccomp.ActTrace.SetReturnCode(int16(syscall.EPERM))
 	actErrno = libseccomp.ActErrno.SetReturnCode(int16(syscall.EPERM))
+
+	// SeccompModeFilter refers to the syscall argument SECCOMP_MODE_FILTER.
+	SeccompModeFilter = uintptr(2)
 )
 
 // Filters given syscalls in a container, preventing them from being used
@@ -73,6 +79,24 @@ func InitSeccomp(config *configs.Seccomp) error {
 	return nil
 }
 
+// IsEnabled returns if the kernel has been configured to support seccomp.
+func IsEnabled() bool {
+	// Try to read from /proc/self/status for kernels > 3.8
+	s, err := parseStatusFile("/proc/self/status")
+	if err != nil {
+		// Check if Seccomp is supported, via CONFIG_SECCOMP.
+		if _, _, err := syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_GET_SECCOMP, 0, 0); err != syscall.EINVAL {
+			// Make sure the kernel has CONFIG_SECCOMP_FILTER.
+			if _, _, err := syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_SECCOMP, SeccompModeFilter, 0); err != syscall.EINVAL {
+				return true
+			}
+		}
+		return false
+	}
+	_, ok := s["Seccomp"]
+	return ok
+}
+
 // Convert Libcontainer Action to Libseccomp ScmpAction
 func getAction(act configs.Action) (libseccomp.ScmpAction, error) {
 	switch act {
@@ -178,3 +202,30 @@ func matchCall(filter *libseccomp.ScmpFilter, call *configs.Syscall) error {
 
 	return nil
 }
+
+func parseStatusFile(path string) (map[string]string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	s := bufio.NewScanner(f)
+	status := make(map[string]string)
+
+	for s.Scan() {
+		if err := s.Err(); err != nil {
+			return nil, err
+		}
+
+		text := s.Text()
+		parts := strings.Split(text, ":")
+
+		if len(parts) <= 1 {
+			continue
+		}
+
+		status[parts[0]] = parts[1]
+	}
+	return status, nil
+}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/seccomp/seccomp_unsupported.go

@@ -17,3 +17,8 @@ func InitSeccomp(config *configs.Seccomp) error {
 	}
 	return nil
 }
+
+// IsEnabled returns false, because it is not supported.
+func IsEnabled() bool {
+	return false
+}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/selinux/selinux.go

@@ -231,10 +231,14 @@ func ReserveLabel(scon string) {
 	}
 }
 
+func selinuxEnforcePath() string {
+	return fmt.Sprintf("%s/enforce", selinuxPath)
+}
+
 func SelinuxGetEnforce() int {
 	var enforce int
 
-	enforceS, err := readCon(fmt.Sprintf("%s/enforce", selinuxPath))
+	enforceS, err := readCon(selinuxEnforcePath())
 	if err != nil {
 		return -1
 	}
@@ -246,6 +250,10 @@ func SelinuxGetEnforce() int {
 	return enforce
 }
 
+func SelinuxSetEnforce(mode int) error {
+	return writeCon(selinuxEnforcePath(), fmt.Sprintf("%d", mode))
+}
+
 func SelinuxGetEnforceMode() int {
 	switch readConfig(selinuxTag) {
 	case "enforcing":

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/standard_init_linux.go

@@ -3,6 +3,7 @@
 package libcontainer
 
 import (
+	"io"
 	"os"
 	"syscall"
 
@@ -14,6 +15,7 @@ import (
 )
 
 type linuxStandardInit struct {
+	pipe      io.ReadWriter
 	parentPid int
 	config    *initConfig
 }
@@ -50,7 +52,6 @@ func (l *linuxStandardInit) Init() error {
 	if err := setOomScoreAdj(l.config.Config.OomScoreAdj); err != nil {
 		return err
 	}
-
 	label.Init()
 	// InitializeMountNamespace() can be executed only for a new mount namespace
 	if l.config.Config.Namespaces.Contains(configs.NEWNS) {
@@ -75,7 +76,6 @@ func (l *linuxStandardInit) Init() error {
 			return err
 		}
 	}
-
 	for _, path := range l.config.Config.ReadonlyPaths {
 		if err := remountReadonly(path); err != nil {
 			return err
@@ -90,6 +90,12 @@ func (l *linuxStandardInit) Init() error {
 	if err != nil {
 		return err
 	}
+	// Tell our parent that we're ready to Execv. This must be done before the
+	// Seccomp rules have been applied, because we need to be able to read and
+	// write to a socket.
+	if err := syncParentReady(l.pipe); err != nil {
+		return err
+	}
 	if l.config.Config.Seccomp != nil {
 		if err := seccomp.InitSeccomp(l.config.Config.Seccomp); err != nil {
 			return err

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/state_linux.go

@@ -0,0 +1,223 @@
+// +build linux
+
+package libcontainer
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+func newStateTransitionError(from, to containerState) error {
+	return &stateTransitionError{
+		From: from.status().String(),
+		To:   to.status().String(),
+	}
+}
+
+// stateTransitionError is returned when an invalid state transition happens from one
+// state to another.
+type stateTransitionError struct {
+	From string
+	To   string
+}
+
+func (s *stateTransitionError) Error() string {
+	return fmt.Sprintf("invalid state transition from %s to %s", s.From, s.To)
+}
+
+type containerState interface {
+	transition(containerState) error
+	destroy() error
+	status() Status
+}
+
+func destroy(c *linuxContainer) error {
+	if !c.config.Namespaces.Contains(configs.NEWPID) {
+		if err := killCgroupProcesses(c.cgroupManager); err != nil {
+			logrus.Warn(err)
+		}
+	}
+	err := c.cgroupManager.Destroy()
+	if rerr := os.RemoveAll(c.root); err == nil {
+		err = rerr
+	}
+	c.initProcess = nil
+	if herr := runPoststopHooks(c); err == nil {
+		err = herr
+	}
+	c.state = &stoppedState{c: c}
+	return err
+}
+
+func runPoststopHooks(c *linuxContainer) error {
+	if c.config.Hooks != nil {
+		s := configs.HookState{
+			Version: c.config.Version,
+			ID:      c.id,
+			Root:    c.config.Rootfs,
+		}
+		for _, hook := range c.config.Hooks.Poststop {
+			if err := hook.Run(s); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// stoppedState represents a container is a stopped/destroyed state.
+type stoppedState struct {
+	c *linuxContainer
+}
+
+func (b *stoppedState) status() Status {
+	return Destroyed
+}
+
+func (b *stoppedState) transition(s containerState) error {
+	switch s.(type) {
+	case *runningState:
+		b.c.state = s
+		return nil
+	case *restoredState:
+		b.c.state = s
+		return nil
+	case *stoppedState:
+		return nil
+	}
+	return newStateTransitionError(b, s)
+}
+
+func (b *stoppedState) destroy() error {
+	return destroy(b.c)
+}
+
+// runningState represents a container that is currently running.
+type runningState struct {
+	c *linuxContainer
+}
+
+func (r *runningState) status() Status {
+	return Running
+}
+
+func (r *runningState) transition(s containerState) error {
+	switch s.(type) {
+	case *stoppedState:
+		running, err := r.c.isRunning()
+		if err != nil {
+			return err
+		}
+		if running {
+			return newGenericError(fmt.Errorf("container still running"), ContainerNotStopped)
+		}
+		r.c.state = s
+		return nil
+	case *pausedState, *nullState:
+		r.c.state = s
+		return nil
+	case *runningState:
+		return nil
+	}
+	return newStateTransitionError(r, s)
+}
+
+func (r *runningState) destroy() error {
+	running, err := r.c.isRunning()
+	if err != nil {
+		return err
+	}
+	if running {
+		return newGenericError(fmt.Errorf("container is not destroyed"), ContainerNotStopped)
+	}
+	return destroy(r.c)
+}
+
+// pausedState represents a container that is currently pause.  It cannot be destroyed in a
+// paused state and must transition back to running first.
+type pausedState struct {
+	c *linuxContainer
+}
+
+func (p *pausedState) status() Status {
+	return Paused
+}
+
+func (p *pausedState) transition(s containerState) error {
+	switch s.(type) {
+	case *runningState, *stoppedState:
+		p.c.state = s
+		return nil
+	case *pausedState:
+		return nil
+	}
+	return newStateTransitionError(p, s)
+}
+
+func (p *pausedState) destroy() error {
+	isRunning, err := p.c.isRunning()
+	if err != nil {
+		return err
+	}
+	if !isRunning {
+		if err := p.c.cgroupManager.Freeze(configs.Thawed); err != nil {
+			return err
+		}
+		return destroy(p.c)
+	}
+	return newGenericError(fmt.Errorf("container is paused"), ContainerPaused)
+}
+
+// restoredState is the same as the running state but also has accociated checkpoint
+// information that maybe need destroyed when the container is stopped and destory is called.
+type restoredState struct {
+	imageDir string
+	c        *linuxContainer
+}
+
+func (r *restoredState) status() Status {
+	return Running
+}
+
+func (r *restoredState) transition(s containerState) error {
+	switch s.(type) {
+	case *stoppedState:
+		return nil
+	case *runningState:
+		return nil
+	}
+	return newStateTransitionError(r, s)
+}
+
+func (r *restoredState) destroy() error {
+	if _, err := os.Stat(filepath.Join(r.c.root, "checkpoint")); err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+	}
+	return destroy(r.c)
+}
+
+// nullState is used whenever a container is restored, loaded, or setting additional
+// processes inside and it should not be destroyed when it is exiting.
+type nullState struct {
+	c *linuxContainer
+	s Status
+}
+
+func (n *nullState) status() Status {
+	return n.s
+}
+
+func (n *nullState) transition(s containerState) error {
+	n.c.state = s
+	return nil
+}
+
+func (n *nullState) destroy() error {
+	return nil
+}

Godeps/_workspace/src/github.com/opencontainers/runc/libcontainer/system/linux.go

@@ -3,6 +3,9 @@
 package system
 
 import (
+	"bufio"
+	"fmt"
+	"os"
 	"os/exec"
 	"syscall"
 	"unsafe"
@@ -75,3 +78,37 @@ func Setctty() error {
 	}
 	return nil
 }
+
+/*
+ * Detect whether we are currently running in a user namespace.
+ * Copied from github.com/lxc/lxd/shared/util.go
+ */
+func RunningInUserNS() bool {
+	file, err := os.Open("/proc/self/uid_map")
+	if err != nil {
+		/*
+		 * This kernel-provided file only exists if user namespaces are
+		 * supported
+		 */
+		return false
+	}
+	defer file.Close()
+
+	buf := bufio.NewReader(file)
+	l, _, err := buf.ReadLine()
+	if err != nil {
+		return false
+	}
+
+	line := string(l)
+	var a, b, c int64
+	fmt.Sscanf(line, "%d %d %d", &a, &b, &c)
+	/*
+	 * We assume we are in the initial user namespace if we have a full
+	 * range - 4294967295 uids starting at uid 0.
+	 */
+	if a == 0 && b == 0 && c == 4294967295 {
+		return false
+	}
+	return true
+}

container/docker/handler.go

@@ -206,20 +206,23 @@ func libcontainerConfigToContainerSpec(config *libcontainerconfigs.Config, mi *i
 	spec.HasMemory = true
 	spec.Memory.Limit = math.MaxUint64
 	spec.Memory.SwapLimit = math.MaxUint64
-	if config.Cgroups.Memory > 0 {
+
-		spec.Memory.Limit = uint64(config.Cgroups.Memory)
+	if config.Cgroups.Resources != nil {
+		if config.Cgroups.Resources.Memory > 0 {
+			spec.Memory.Limit = uint64(config.Cgroups.Resources.Memory)
 		}
-	if config.Cgroups.MemorySwap > 0 {
+		if config.Cgroups.Resources.MemorySwap > 0 {
-		spec.Memory.SwapLimit = uint64(config.Cgroups.MemorySwap)
+			spec.Memory.SwapLimit = uint64(config.Cgroups.Resources.MemorySwap)
 		}
 
 		// Get CPU info
 		spec.HasCpu = true
 		spec.Cpu.Limit = 1024
-	if config.Cgroups.CpuShares != 0 {
+		if config.Cgroups.Resources.CpuShares != 0 {
-		spec.Cpu.Limit = uint64(config.Cgroups.CpuShares)
+			spec.Cpu.Limit = uint64(config.Cgroups.Resources.CpuShares)
+		}
+		spec.Cpu.Mask = utils.FixCpuMask(config.Cgroups.Resources.CpusetCpus, mi.NumCores)
 	}
-	spec.Cpu.Mask = utils.FixCpuMask(config.Cgroups.CpusetCpus, mi.NumCores)
 
 	spec.HasDiskIo = true
 

container/libcontainer/compatibility.go

@@ -294,29 +294,30 @@ func convertOldConfigToNew(config v1Config) *configs.Config {
 	var newCgroup = &configs.Cgroup{
 		Name:   old.Name,
 		Parent: old.Parent,
-		AllowAllDevices:   old.AllowAllDevices,
+		Resources: &configs.Resources{
-		AllowedDevices:    old.AllowedDevices,
+			AllowAllDevices:   old.Resources.AllowAllDevices,
-		DeniedDevices:     old.DeniedDevices,
+			AllowedDevices:    old.Resources.AllowedDevices,
-		Memory:            old.Memory,
+			DeniedDevices:     old.Resources.DeniedDevices,
-		MemoryReservation: old.MemoryReservation,
+			Memory:            old.Resources.Memory,
-		MemorySwap:        old.MemorySwap,
+			MemoryReservation: old.Resources.MemoryReservation,
-		KernelMemory:      old.KernelMemory,
+			MemorySwap:        old.Resources.MemorySwap,
-		CpuShares:         old.CpuShares,
+			KernelMemory:      old.Resources.KernelMemory,
-		CpuQuota:          old.CpuQuota,
+			CpuShares:         old.Resources.CpuShares,
-		CpuPeriod:         old.CpuPeriod,
+			CpuQuota:          old.Resources.CpuQuota,
-		CpuRtRuntime:      old.CpuRtRuntime,
+			CpuPeriod:         old.Resources.CpuPeriod,
-		CpuRtPeriod:       old.CpuRtPeriod,
+			CpuRtRuntime:      old.Resources.CpuRtRuntime,
-		CpusetCpus:        old.CpusetCpus,
+			CpuRtPeriod:       old.Resources.CpuRtPeriod,
-		CpusetMems:        old.CpusetMems,
+			CpusetCpus:        old.Resources.CpusetCpus,
-		BlkioWeight:       old.BlkioWeight,
+			CpusetMems:        old.Resources.CpusetMems,
-		BlkioLeafWeight:   old.BlkioLeafWeight,
+			BlkioWeight:       old.Resources.BlkioWeight,
-		Freezer:           old.Freezer,
+			BlkioLeafWeight:   old.Resources.BlkioLeafWeight,
-		HugetlbLimit:      old.HugetlbLimit,
+			Freezer:           old.Resources.Freezer,
-		Slice:             old.Slice,
+			HugetlbLimit:      old.Resources.HugetlbLimit,
-		OomKillDisable:    old.OomKillDisable,
+			OomKillDisable:    old.Resources.OomKillDisable,
-		MemorySwappiness:  old.MemorySwappiness,
+			MemorySwappiness:  old.Resources.MemorySwappiness,
-		NetPrioIfpriomap:  old.NetPrioIfpriomap,
+			NetPrioIfpriomap:  old.Resources.NetPrioIfpriomap,
-		NetClsClassid:     old.NetClsClassid,
+			NetClsClassid:     old.Resources.NetClsClassid,
+		},
 	}
 
 	result.Cgroups = newCgroup