first cut of rkt handler

This commit is contained in:
Shaya Potter 2016-03-07 15:19:57 -08:00
parent b9e36443c4
commit 206670a655
6 changed files with 635 additions and 7 deletions

65
container/rkt/client.go Normal file
View File

@ -0,0 +1,65 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rkt
import (
"fmt"
"sync"
"time"
rktapi "github.com/coreos/rkt/api/v1alpha"
"golang.org/x/net/context"
"google.golang.org/grpc"
)
const (
defaultRktAPIServiceAddr = "localhost:15441"
)
var (
rktClient rktapi.PublicAPIClient
rktClientErr error
once sync.Once
)
func Client() (rktapi.PublicAPIClient, error) {
once.Do(func() {
apisvcConn, err := grpc.Dial(defaultRktAPIServiceAddr, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second))
if err != nil {
rktClient = nil
rktClientErr = fmt.Errorf("rkt: cannot connect to rkt api service: %v", err)
return
}
rktClient = rktapi.NewPublicAPIClient(apisvcConn)
})
return rktClient, rktClientErr
}
func RktPath() (string, error) {
client, err := Client()
if err != nil {
return "", err
}
resp, err := client.GetInfo(context.Background(), &rktapi.GetInfoRequest{})
if err != nil {
return "", fmt.Errorf("couldn't GetInfo from rkt api servie: %v", err)
}
return resp.Info.GlobalFlags.Dir, nil
}

106
container/rkt/factory.go Normal file
View File

@ -0,0 +1,106 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rkt
import (
"fmt"
"strings"
rktapi "github.com/coreos/rkt/api/v1alpha"
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/container/libcontainer"
"github.com/google/cadvisor/fs"
info "github.com/google/cadvisor/info/v1"
"golang.org/x/net/context"
"github.com/golang/glog"
)
const RktNamespace = "rkt"
type rktFactory struct {
machineInfoFactory info.MachineInfoFactory
cgroupSubsystems *libcontainer.CgroupSubsystems
fsInfo fs.FsInfo
ignoreMetrics container.MetricSet
rktPath string
}
func (self *rktFactory) String() string {
return "rkt"
}
func (self *rktFactory) NewContainerHandler(name string, inHostNamespace bool) (container.ContainerHandler, error) {
client, err := Client()
if err != nil {
return nil, err
}
rootFs := "/"
if !inHostNamespace {
rootFs = "/rootfs"
}
return newRktContainerHandler(name, client, self.rktPath, self.cgroupSubsystems, self.machineInfoFactory, self.fsInfo, rootFs, self.ignoreMetrics)
}
func (self *rktFactory) CanHandleAndAccept(name string) (bool, bool, error) {
// TODO{SJP}: will ignore all cgroup names that don't either correspond to the machine.slice that is the pod
// or the containers that belong to the pod
if strings.HasPrefix(name, "/machine.slice/machine-rkt\\x2d") {
accept, err := verifyName(name)
return true, accept, err
}
return false, false, fmt.Errorf("%s not handled by rkt handler", name)
}
func (self *rktFactory) DebugInfo() map[string][]string {
return map[string][]string{}
}
func Register(machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, ignoreMetrics container.MetricSet) error {
client, err := Client()
if err != nil {
return fmt.Errorf("unable to communicate with Rkt api service: %v", err)
}
resp, err := client.GetInfo(context.Background(), &rktapi.GetInfoRequest{})
if err != nil {
return fmt.Errorf("couldn't GetInfo from rkt api servie: %v", err)
}
rktPath := resp.Info.GlobalFlags.Dir
cgroupSubsystems, err := libcontainer.GetCgroupSubsystems()
if err != nil {
return fmt.Errorf("failed to get cgroup subsystems: %v", err)
}
if len(cgroupSubsystems.Mounts) == 0 {
return fmt.Errorf("failed to find supported cgroup mounts for the raw factory")
}
glog.Infof("Registering Rkt factory")
factory := &rktFactory{
machineInfoFactory: machineInfoFactory,
fsInfo: fsInfo,
cgroupSubsystems: &cgroupSubsystems,
ignoreMetrics: ignoreMetrics,
rktPath: rktPath,
}
container.RegisterContainerHandlerFactory(factory)
return nil
}

339
container/rkt/handler.go Normal file
View File

@ -0,0 +1,339 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Handler for "rkt" containers.
package rkt
import (
"fmt"
"os"
"path"
"time"
rktapi "github.com/coreos/rkt/api/v1alpha"
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/container/common"
"github.com/google/cadvisor/container/libcontainer"
"github.com/google/cadvisor/fs"
info "github.com/google/cadvisor/info/v1"
"github.com/google/cadvisor/utils"
"golang.org/x/net/context"
"github.com/golang/glog"
"github.com/opencontainers/runc/libcontainer/cgroups"
cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
"github.com/opencontainers/runc/libcontainer/configs"
)
type rktContainerHandler struct {
rktClient rktapi.PublicAPIClient
// Name of the container for this handler.
name string
cgroupSubsystems *libcontainer.CgroupSubsystems
machineInfoFactory info.MachineInfoFactory
// Absolute path to the cgroup hierarchies of this container.
// (e.g.: "cpu" -> "/sys/fs/cgroup/cpu/test")
cgroupPaths map[string]string
// Manager of this container's cgroups.
cgroupManager cgroups.Manager
// Whether this container has network isolation enabled.
hasNetwork bool
fsInfo fs.FsInfo
externalMounts []common.Mount
rootFs string
isPod bool
aliases []string
pid int
rootfsStorageDir string
// Filesystem handler.
fsHandler common.FsHandler
ignoreMetrics container.MetricSet
}
func (self *rktContainerHandler) GetCgroupPaths() map[string]string {
return self.cgroupPaths
}
func (self *rktContainerHandler) GetMachineInfoFactory() info.MachineInfoFactory {
return self.machineInfoFactory
}
func (self *rktContainerHandler) GetName() string {
return self.name
}
func (self *rktContainerHandler) GetExternalMounts() []common.Mount {
return self.externalMounts
}
func (self *rktContainerHandler) HasNetwork() bool {
return self.hasNetwork && !self.ignoreMetrics.Has(container.NetworkUsageMetrics)
}
func (self *rktContainerHandler) HasFilesystem() bool {
if !self.ignoreMetrics.Has(container.DiskUsageMetrics) {
return true
}
return false
}
func newRktContainerHandler(name string, rktClient rktapi.PublicAPIClient, rktPath string, cgroupSubsystems *libcontainer.CgroupSubsystems, machineInfoFactory info.MachineInfoFactory, fsInfo fs.FsInfo, rootFs string, ignoreMetrics container.MetricSet) (container.ContainerHandler, error) {
aliases := make([]string, 1)
isPod := false
parsed, err := parseName(name)
if err != nil {
return nil, fmt.Errorf("this should be impossible!, new handler failing, but factory allowed, name = %s", name)
}
//rktnetes uses containerID: rkt://fff40827-b994-4e3a-8f88-6427c2c8a5ac:nginx
if parsed.Container == "" {
isPod = true
aliases = append(aliases, "rkt://"+parsed.Pod)
} else {
aliases = append(aliases, "rkt://"+parsed.Pod+":"+parsed.Container)
}
pid := os.Getpid()
if parsed.Container == "" {
resp, err := rktClient.InspectPod(context.Background(), &rktapi.InspectPodRequest{
Id: parsed.Pod,
})
if err != nil {
return nil, err
}
pid = int(resp.Pod.Pid)
} else {
glog.Infof("skipping as Container")
}
cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems.MountPoints, name)
cHints, err := common.GetContainerHintsFromFile(*common.ArgContainerHints)
if err != nil {
return nil, err
}
// Generate the equivalent cgroup manager for this container.
cgroupManager := &cgroupfs.Manager{
Cgroups: &configs.Cgroup{
Name: name,
},
Paths: cgroupPaths,
}
hasNetwork := false
if isPod {
hasNetwork = true
}
//SJP: unsure the point of this code, if it event does anything today?
var externalMounts []common.Mount
for _, container := range cHints.AllHosts {
if name == container.FullName {
externalMounts = container.Mounts
break
}
}
rootfsStorageDir := getRootFs(rktPath, parsed)
handler := &rktContainerHandler{
name: name,
rktClient: rktClient,
cgroupSubsystems: cgroupSubsystems,
machineInfoFactory: machineInfoFactory,
cgroupPaths: cgroupPaths,
cgroupManager: cgroupManager,
fsInfo: fsInfo,
hasNetwork: hasNetwork,
externalMounts: externalMounts,
rootFs: rootFs,
isPod: isPod,
aliases: aliases,
pid: pid,
rootfsStorageDir: rootfsStorageDir,
ignoreMetrics: ignoreMetrics,
}
if !ignoreMetrics.Has(container.DiskUsageMetrics) {
handler.fsHandler = common.NewFsHandler(time.Minute, rootfsStorageDir, "", fsInfo)
}
return handler, nil
}
func (self *rktContainerHandler) ContainerReference() (info.ContainerReference, error) {
return info.ContainerReference{
Name: self.name,
Aliases: self.aliases,
Namespace: RktNamespace,
}, nil
}
//SJP: Should a Rkt containe have have htis?
func (self *rktContainerHandler) GetRootNetworkDevices() ([]info.NetInfo, error) {
nd := []info.NetInfo{}
return nd, nil
}
func (self *rktContainerHandler) Start() {
self.fsHandler.Start()
}
func (self *rktContainerHandler) Cleanup() {
self.fsHandler.Stop()
}
func (self *rktContainerHandler) GetSpec() (info.ContainerSpec, error) {
return common.GetSpec(self)
}
func (self *rktContainerHandler) getFsStats(stats *info.ContainerStats) error {
if self.ignoreMetrics.Has(container.DiskUsageMetrics) {
return nil
}
deviceInfo, err := self.fsInfo.GetDirFsDevice(self.rootfsStorageDir)
if err != nil {
return err
}
mi, err := self.machineInfoFactory.GetMachineInfo()
if err != nil {
return err
}
var limit uint64 = 0
// SJP: Docker does not impose any filesystem limits for containers. So it uses capacity as limit.
// Doing the same for Rkt. is this true?
for _, fs := range mi.Filesystems {
if fs.Device == deviceInfo.Device {
limit = fs.Capacity
break
}
}
fsStat := info.FsStats{Device: deviceInfo.Device, Limit: limit}
fsStat.BaseUsage, fsStat.Usage = self.fsHandler.Usage()
stats.Filesystem = append(stats.Filesystem, fsStat)
return nil
}
func (self *rktContainerHandler) GetStats() (*info.ContainerStats, error) {
stats, err := libcontainer.GetStats(self.cgroupManager, self.rootFs, self.pid, self.ignoreMetrics)
if err != nil {
return stats, err
}
// Get filesystem stats.
err = self.getFsStats(stats)
if err != nil {
return stats, err
}
return stats, nil
}
func (self *rktContainerHandler) GetCgroupPath(resource string) (string, error) {
path, ok := self.cgroupPaths[resource]
if !ok {
return "", fmt.Errorf("could not find path for resource %q for container %q\n", resource, self.name)
}
return path, nil
}
//TODO{SJP} need to figure out what to put here
func (self *rktContainerHandler) GetContainerLabels() map[string]string {
return map[string]string{}
}
func (self *rktContainerHandler) ListContainers(listType container.ListType) ([]info.ContainerReference, error) {
containers := make(map[string]struct{})
// Rkt containers do not have subcontainers, only the "Pod" does.
if self.isPod == false {
var ret []info.ContainerReference
return ret, nil
}
// Turn the system.slice cgroups into the Pod's subcontainers
for _, cgroupPath := range self.cgroupPaths {
err := common.ListDirectories(path.Join(cgroupPath, "system.slice"), path.Join(self.name, "system.slice"), listType == container.ListRecursive, containers)
if err != nil {
return nil, err
}
}
// Create the container references. for the Pod's subcontainers
ret := make([]info.ContainerReference, 0, len(containers))
for cont := range containers {
aliases := make([]string, 1)
parsed, err := parseName(cont)
if err != nil {
return nil, fmt.Errorf("this should be impossible!, unable to parse rkt subcontainer name = %s", cont)
}
aliases = append(aliases, parsed.Pod+":"+parsed.Container)
ret = append(ret, info.ContainerReference{
Name: cont,
Aliases: aliases,
Namespace: RktNamespace,
})
}
return ret, nil
}
func (self *rktContainerHandler) ListThreads(listType container.ListType) ([]int, error) {
// TODO(vmarmol): Implement
return nil, nil
}
func (self *rktContainerHandler) ListProcesses(listType container.ListType) ([]int, error) {
return libcontainer.GetProcesses(self.cgroupManager)
}
func (self *rktContainerHandler) WatchSubcontainers(events chan container.SubcontainerEvent) error {
return fmt.Errorf("watch is unimplemented in the Rkt container driver")
}
func (self *rktContainerHandler) StopWatchingSubcontainers() error {
// No-op for Rkt driver.
return nil
}
func (self *rktContainerHandler) Exists() bool {
// If any cgroup exists, the container is still alive.
for _, cgroupPath := range self.cgroupPaths {
if utils.FileExists(cgroupPath) {
return true
}
}
return false
}

92
container/rkt/helpers.go Normal file
View File

@ -0,0 +1,92 @@
// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rkt
import (
"fmt"
"io/ioutil"
"path"
"strings"
"github.com/golang/glog"
)
type parsedName struct {
Pod string
Container string
}
func verifyName(name string) (bool, error) {
_, err := parseName(name)
return err == nil, err
}
/* Parse cgroup name into a pod/container name struct
Example cgroup fs name
pod - /sys/fs/cgroup/cpu/machine.slice/machine-rkt\\x2df556b64a\\x2d17a7\\x2d47d7\\x2d93ec\\x2def2275c3d67e.scope/
container under pod - /sys/fs/cgroup/cpu/machine.slice/machine-rkt\\x2df556b64a\\x2d17a7\\x2d47d7\\x2d93ec\\x2def2275c3d67e.scope/system.slice/alpine-sh.service
*/
func parseName(name string) (*parsedName, error) {
splits := strings.Split(name, "/")
if len(splits) == 3 || len(splits) == 5 {
parsed := &parsedName{}
if splits[1] == "machine.slice" {
replacer := strings.NewReplacer("machine-rkt\\x2d", "", ".scope", "", "\\x2d", "-")
parsed.Pod = replacer.Replace(splits[2])
if len(splits) == 3 {
return parsed, nil
}
if splits[3] == "system.slice" {
parsed.Container = strings.Replace(splits[4], ".service", "", -1)
return parsed, nil
}
}
}
return nil, fmt.Errorf("%s not handled by rkt handler", name)
}
// Gets a Rkt container's overlay upper dir
func getRootFs(root string, parsed *parsedName) string {
/* Example of where it stores the upper dir key
for container
/var/lib/rkt/pods/run/bc793ec6-c48f-4480-99b5-6bec16d52210/appsinfo/alpine-sh/treeStoreID
for pod
/var/lib/rkt/pods/run/f556b64a-17a7-47d7-93ec-ef2275c3d67e/stage1TreeStoreID
*/
var tree string
if parsed.Container == "" {
tree = path.Join(root, "pods/run", parsed.Pod, "stage1TreeStoreID")
} else {
tree = path.Join(root, "pods/run", parsed.Pod, "appsinfo", parsed.Container, "treeStoreID")
}
bytes, err := ioutil.ReadFile(tree)
if err != nil {
glog.Infof("ReadFile failed: %v", err)
return ""
}
s := string(bytes)
/* Example of where the upper dir is stored via key read above
/var/lib/rkt/pods/run/bc793ec6-c48f-4480-99b5-6bec16d52210/overlay/deps-sha512-82a099e560a596662b15dec835e9adabab539cad1f41776a30195a01a8f2f22b/
*/
return path.Join(root, "pods/run", parsed.Pod, "overlay", s)
}

View File

@ -40,6 +40,7 @@ import (
const (
LabelSystemRoot = "root"
LabelDockerImages = "docker-images"
LabelRktImages = "rkt-images"
)
type partition struct {
@ -64,6 +65,7 @@ type Context struct {
// docker root directory.
DockerRoot string
DockerInfo map[string]string
RktPath string
}
func NewFsInfo(context Context) (FsInfo, error) {
@ -79,6 +81,7 @@ func NewFsInfo(context Context) (FsInfo, error) {
fsInfo.addSystemRootLabel(mounts)
fsInfo.addDockerImagesLabel(context, mounts)
fsInfo.addRktImagesLabel(context, mounts)
supportedFsType := map[string]bool{
// all ext systems are checked through prefix.
@ -168,7 +171,21 @@ func (self *RealFsInfo) addDockerImagesLabel(context Context, mounts []*mount.In
self.partitions[dockerDev] = *dockerPartition
self.labels[LabelDockerImages] = dockerDev
} else {
self.updateDockerImagesPath(mounts, getDockerImagePaths(context))
self.updateContainerImagesPath(LabelDockerImages, mounts, getDockerImagePaths(context))
}
}
func (self *RealFsInfo) addRktImagesLabel(context Context, mounts []*mount.Info) {
if context.RktPath != "" {
rktPath := context.RktPath
rktImagesPaths := map[string]struct{}{
"/": {},
}
for rktPath != "/" && rktPath != "." {
rktImagesPaths[rktPath] = struct{}{}
rktPath = filepath.Dir(rktPath)
}
self.updateContainerImagesPath(LabelRktImages, mounts, rktImagesPaths)
}
}
@ -194,10 +211,10 @@ func getDockerImagePaths(context Context) map[string]struct{} {
// This method compares the mountpoints with possible docker image mount points. If a match is found,
// docker images label is added to the partition.
func (self *RealFsInfo) updateDockerImagesPath(mounts []*mount.Info, dockerImagePaths map[string]struct{}) {
func (self *RealFsInfo) updateContainerImagesPath(label string, mounts []*mount.Info, containerImagePaths map[string]struct{}) {
var useMount *mount.Info
for _, m := range mounts {
if _, ok := dockerImagePaths[m.Mountpoint]; ok {
if _, ok := containerImagePaths[m.Mountpoint]; ok {
if useMount == nil || (len(useMount.Mountpoint) < len(m.Mountpoint)) {
useMount = m
}
@ -210,7 +227,7 @@ func (self *RealFsInfo) updateDockerImagesPath(mounts []*mount.Info, dockerImage
major: uint(useMount.Major),
minor: uint(useMount.Minor),
}
self.labels[LabelDockerImages] = useMount.Source
self.labels[label] = useMount.Source
}
}

View File

@ -31,6 +31,7 @@ import (
"github.com/google/cadvisor/container"
"github.com/google/cadvisor/container/docker"
"github.com/google/cadvisor/container/raw"
"github.com/google/cadvisor/container/rkt"
"github.com/google/cadvisor/events"
"github.com/google/cadvisor/fs"
info "github.com/google/cadvisor/info/v1"
@ -135,7 +136,12 @@ func New(memoryCache *memory.InMemoryCache, sysfs sysfs.SysFs, maxHousekeepingIn
if err != nil {
glog.Warningf("Unable to connect to Docker: %v", err)
}
context := fs.Context{DockerRoot: docker.RootDir(), DockerInfo: dockerInfo}
rktPath, err := rkt.RktPath()
if err != nil {
glog.Warningf("unable to connect to Rkt api service: %v", err)
}
context := fs.Context{DockerRoot: docker.RootDir(), DockerInfo: dockerInfo, RktPath: rktPath}
fsInfo, err := fs.NewFsInfo(context)
if err != nil {
return nil, err
@ -206,13 +212,16 @@ type manager struct {
// Start the container manager.
func (self *manager) Start() error {
// Register Docker container factory.
err := docker.Register(self, self.fsInfo, self.ignoreMetrics)
if err != nil {
glog.Errorf("Docker container factory registration failed: %v.", err)
}
// Register the raw driver.
err = rkt.Register(self, self.fsInfo, self.ignoreMetrics)
if err != nil {
glog.Errorf("Registration of the rkt container factory failed: %v", err)
}
err = raw.Register(self, self.fsInfo, self.ignoreMetrics)
if err != nil {
glog.Errorf("Registration of the raw container factory failed: %v", err)