Merge pull request #384 from rjnagal/cpu

Add cache info to machine topology

info/machine.go

@@ -27,11 +27,22 @@ type Node struct {
 	// Per-node memory
 	Memory uint64  `json:"memory"`
 	Cores  []Core  `json:"cores"`
+	Caches []Cache `json:"caches"`
 }
 
 type Core struct {
 	Id      int     `json:"core_id"`
 	Threads []int   `json:"thread_ids"`
+	Caches  []Cache `json:"caches"`
+}
+
+type Cache struct {
+	// Size of memory cache in bytes.
+	Size uint64 `json:"size"`
+	// Type of memory cache: data, instruction, or unified.
+	Type string `json:"type"`
+	// Level (distance from cpus) in a multi-level cache hierarchy.
+	Level int `json:"level"`
 }
 
 func (self *Node) FindCore(id int) (bool, int) {
@@ -60,6 +71,16 @@ func (self *Node) AddThread(thread int, core int) {
 	self.Cores[coreIdx].Threads = append(self.Cores[coreIdx].Threads, thread)
 }
 
+func (self *Node) AddNodeCache(c Cache) {
+	self.Caches = append(self.Caches, c)
+}
+
+func (self *Node) AddPerCoreCache(c Cache) {
+	for idx, _ := range self.Cores {
+		self.Cores[idx].Caches = append(self.Cores[idx].Caches, c)
+	}
+}
+
 type DiskInfo struct {
 	// device name
 	Name string `json:"name"`

manager/machine.go

@@ -128,7 +128,7 @@ func addNode(nodes *[]info.Node, id int) (int, error) {
 	return idx, nil
 }
 
-func getTopology(cpuinfo string) ([]info.Node, int, error) {
+func getTopology(sysFs sysfs.SysFs, cpuinfo string) ([]info.Node, int, error) {
 	nodes := []info.Node{}
 	numCores := 0
 	lastThread := -1
@@ -177,6 +177,29 @@ func getTopology(cpuinfo string) ([]info.Node, int, error) {
 	if numCores < 1 {
 		return nil, numCores, fmt.Errorf("could not detect any cores")
 	}
+	for idx, node := range nodes {
+		caches, err := sysfs.GetCacheInfo(sysFs, node.Cores[0].Id)
+		if err != nil {
+			return nil, -1, fmt.Errorf("failed to get cache information for node %d: %v", node.Id, err)
+		}
+		numThreadsPerCore := len(node.Cores[0].Threads)
+		numThreadsPerNode := len(node.Cores) * numThreadsPerCore
+		for _, cache := range caches {
+			c := info.Cache{
+				Size:  cache.Size,
+				Level: cache.Level,
+				Type:  cache.Type,
+			}
+			if cache.Cpus == numThreadsPerNode && cache.Level > 2 {
+				// Add a node-level cache.
+				nodes[idx].AddNodeCache(c)
+			} else if cache.Cpus == numThreadsPerCore {
+				// Add to each core.
+				nodes[idx].AddPerCoreCache(c)
+			}
+			// Ignore unknown caches.
+		}
+	}
 	return nodes, numCores, nil
 }
 
@@ -212,7 +235,7 @@ func getMachineInfo(sysFs sysfs.SysFs) (*info.MachineInfo, error) {
 		return nil, err
 	}
 
-	topology, numCores, err := getTopology(string(cpuinfo))
+	topology, numCores, err := getTopology(sysFs, string(cpuinfo))
 	if err != nil {
 		return nil, err
 	}

manager/topology_test.go

@@ -20,6 +20,8 @@ import (
 	"testing"
 
 	"github.com/google/cadvisor/info"
+	"github.com/google/cadvisor/utils/sysfs"
+	"github.com/google/cadvisor/utils/sysfs/fakesysfs"
 )
 
 func TestTopology(t *testing.T) {
@@ -28,7 +30,15 @@ func TestTopology(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable to read input test file %s", testfile)
 	}
-	topology, numCores, err := getTopology(string(testcpuinfo))
+	sysFs := &fakesysfs.FakeSysFs{}
+	c := sysfs.CacheInfo{
+		Size:  32 * 1024,
+		Type:  "unified",
+		Level: 1,
+		Cpus:  2,
+	}
+	sysFs.SetCacheInfo(c)
+	topology, numCores, err := getTopology(sysFs, string(testcpuinfo))
 	if err != nil {
 		t.Errorf("failed to get topology for sample cpuinfo %s", string(testcpuinfo))
 	}
@@ -40,12 +50,18 @@ func TestTopology(t *testing.T) {
 	numNodes := 2
 	numCoresPerNode := 3
 	numThreads := 2
+	cache := info.Cache{
+		Size:  32 * 1024,
+		Type:  "unified",
+		Level: 1,
+	}
 	for i := 0; i < numNodes; i++ {
 		node := info.Node{Id: i}
 		// Copy over Memory from result. TODO(rjnagal): Use memory from fake.
 		node.Memory = topology[i].Memory
 		for j := 0; j < numCoresPerNode; j++ {
 			core := info.Core{Id: i*numCoresPerNode + j}
+			core.Caches = append(core.Caches, cache)
 			for k := 0; k < numThreads; k++ {
 				core.Threads = append(core.Threads, k*numCoresPerNode*numNodes+core.Id)
 			}
@@ -60,13 +76,27 @@ func TestTopology(t *testing.T) {
 }
 
 func TestTopologyWithSimpleCpuinfo(t *testing.T) {
-	topology, numCores, err := getTopology("processor\t: 0\n")
+	sysFs := &fakesysfs.FakeSysFs{}
+	c := sysfs.CacheInfo{
+		Size:  32 * 1024,
+		Type:  "unified",
+		Level: 1,
+		Cpus:  1,
+	}
+	sysFs.SetCacheInfo(c)
+	topology, numCores, err := getTopology(sysFs, "processor\t: 0\n")
 	if err != nil {
 		t.Errorf("Expected cpuinfo with no topology data to succeed.")
 	}
 	node := info.Node{Id: 0}
 	core := info.Core{Id: 0}
 	core.Threads = append(core.Threads, 0)
+	cache := info.Cache{
+		Size:  32 * 1024,
+		Type:  "unified",
+		Level: 1,
+	}
+	core.Caches = append(core.Caches, cache)
 	node.Cores = append(node.Cores, core)
 	// Copy over Memory from result. TODO(rjnagal): Use memory from fake.
 	node.Memory = topology[0].Memory
@@ -80,7 +110,7 @@ func TestTopologyWithSimpleCpuinfo(t *testing.T) {
 }
 
 func TestTopologyEmptyCpuinfo(t *testing.T) {
-	_, _, err := getTopology("")
+	_, _, err := getTopology(&fakesysfs.FakeSysFs{}, "")
 	if err == nil {
 		t.Errorf("Expected empty cpuinfo to fail.")
 	}

utils/sysfs/fakesysfs/fake.go

@@ -17,14 +17,17 @@ package fakesysfs
 import (
 	"os"
 	"time"
+
+	"github.com/google/cadvisor/utils/sysfs"
 )
 
 // If we extend sysfs to support more interfaces, it might be worth making this a mock instead of a fake.
 type FileInfo struct {
+	EntryName string
 }
 
 func (self *FileInfo) Name() string {
-	return "sda"
+	return self.EntryName
 }
 
 func (self *FileInfo) Size() int64 {
@@ -49,9 +52,11 @@ func (self *FileInfo) Sys() interface{} {
 
 type FakeSysFs struct {
 	info  FileInfo
+	cache sysfs.CacheInfo
 }
 
 func (self *FakeSysFs) GetBlockDevices() ([]os.FileInfo, error) {
+	self.info.EntryName = "sda"
 	return []os.FileInfo{&self.info}, nil
 }
 
@@ -62,3 +67,16 @@ func (self *FakeSysFs) GetBlockDeviceSize(name string) (string, error) {
 func (self *FakeSysFs) GetBlockDeviceNumbers(name string) (string, error) {
 	return "8:0\n", nil
 }
+
+func (self *FakeSysFs) GetCaches(id int) ([]os.FileInfo, error) {
+	self.info.EntryName = "index0"
+	return []os.FileInfo{&self.info}, nil
+}
+
+func (self *FakeSysFs) GetCacheInfo(cpu int, cache string) (sysfs.CacheInfo, error) {
+	return self.cache, nil
+}
+
+func (self *FakeSysFs) SetCacheInfo(cache sysfs.CacheInfo) {
+	self.cache = cache
+}

utils/sysfs/sysfs.go

@@ -25,7 +25,10 @@ import (
 	"github.com/google/cadvisor/info"
 )
 
-const BlockDir = "/sys/block"
+const (
+	blockDir = "/sys/block"
+	cacheDir = "/sys/devices/system/cpu/cpu"
+)
 
 // Abstracts the lowest level calls to sysfs.
 type SysFs interface {
@@ -35,6 +38,11 @@ type SysFs interface {
 	GetBlockDeviceSize(string) (string, error)
 	// Get device major:minor number string.
 	GetBlockDeviceNumbers(string) (string, error)
+
+	// Get directory information for available caches accessible to given cpu.
+	GetCaches(id int) ([]os.FileInfo, error)
+	// Get information for a cache accessible from the given cpu.
+	GetCacheInfo(cpu int, cache string) (CacheInfo, error)
 }
 
 type realSysFs struct{}
@@ -44,11 +52,11 @@ func NewRealSysFs() (SysFs, error) {
 }
 
 func (self *realSysFs) GetBlockDevices() ([]os.FileInfo, error) {
-	return ioutil.ReadDir(BlockDir)
+	return ioutil.ReadDir(blockDir)
 }
 
 func (self *realSysFs) GetBlockDeviceNumbers(name string) (string, error) {
-	dev, err := ioutil.ReadFile(path.Join(BlockDir, name, "/dev"))
+	dev, err := ioutil.ReadFile(path.Join(blockDir, name, "/dev"))
 	if err != nil {
 		return "", err
 	}
@@ -56,13 +64,59 @@ func (self *realSysFs) GetBlockDeviceNumbers(name string) (string, error) {
 }
 
 func (self *realSysFs) GetBlockDeviceSize(name string) (string, error) {
-	size, err := ioutil.ReadFile(path.Join(BlockDir, name, "/size"))
+	size, err := ioutil.ReadFile(path.Join(blockDir, name, "/size"))
 	if err != nil {
 		return "", err
 	}
 	return string(size), nil
 }
 
+func (self *realSysFs) GetCaches(id int) ([]os.FileInfo, error) {
+	cpuPath := fmt.Sprintf("%s%d/cache", cacheDir, id)
+	return ioutil.ReadDir(cpuPath)
+}
+
+func (self *realSysFs) GetCacheInfo(id int, name string) (CacheInfo, error) {
+	cachePath := fmt.Sprintf("%s%d/cache/%s", cacheDir, id, name)
+	out, err := ioutil.ReadFile(path.Join(cachePath, "/size"))
+	if err != nil {
+		return CacheInfo{}, err
+	}
+	var size uint64
+	n, err := fmt.Sscanf(string(out), "%dK", &size)
+	if err != nil || n != 1 {
+		return CacheInfo{}, err
+	}
+	// convert to bytes
+	size = size * 1024
+	out, err = ioutil.ReadFile(path.Join(cachePath, "/level"))
+	if err != nil {
+		return CacheInfo{}, err
+	}
+	var level int
+	n, err = fmt.Sscanf(string(out), "%d", &level)
+	if err != nil || n != 1 {
+		return CacheInfo{}, err
+	}
+
+	out, err = ioutil.ReadFile(path.Join(cachePath, "/type"))
+	if err != nil {
+		return CacheInfo{}, err
+	}
+	cacheType := strings.TrimSpace(string(out))
+	out, err = ioutil.ReadFile(path.Join(cachePath, "/shared_cpu_list"))
+	if err != nil {
+		return CacheInfo{}, err
+	}
+	cpus := strings.Split(string(out), ",")
+	return CacheInfo{
+		Size:  size,
+		Level: level,
+		Type:  cacheType,
+		Cpus:  len(cpus),
+	}, nil
+}
+
 // Get information about block devices present on the system.
 // Uses the passed in system interface to retrieve the low level OS information.
 func GetBlockDeviceInfo(sysfs SysFs) (map[string]info.DiskInfo, error) {
@@ -106,3 +160,34 @@ func GetBlockDeviceInfo(sysfs SysFs) (map[string]info.DiskInfo, error) {
 	}
 	return diskMap, nil
 }
+
+type CacheInfo struct {
+	// size in bytes
+	Size uint64
+	// cache type - instruction, data, unified
+	Type string
+	// distance from cpus in a multi-level hierarchy
+	Level int
+	// number of cpus that can access this cache.
+	Cpus int
+}
+
+func GetCacheInfo(sysfs SysFs, id int) ([]CacheInfo, error) {
+	caches, err := sysfs.GetCaches(id)
+	if err != nil {
+		return nil, err
+	}
+
+	info := []CacheInfo{}
+	for _, cache := range caches {
+		if !strings.HasPrefix(cache.Name(), "index") {
+			continue
+		}
+		cacheInfo, err := sysfs.GetCacheInfo(id, cache.Name())
+		if err != nil {
+			return nil, err
+		}
+		info = append(info, cacheInfo)
+	}
+	return info, nil
+}