diff --git a/info/machine.go b/info/machine.go index c52ff982..998340c0 100644 --- a/info/machine.go +++ b/info/machine.go @@ -25,13 +25,24 @@ type FsInfo struct { type Node struct { Id int `json:"node_id"` // Per-node memory - Memory uint64 `json:"memory"` - Cores []Core `json:"cores"` + Memory uint64 `json:"memory"` + Cores []Core `json:"cores"` + Caches []Cache `json:"caches"` } type Core struct { - Id int `json:"core_id"` - Threads []int `json:"thread_ids"` + Id int `json:"core_id"` + Threads []int `json:"thread_ids"` + Caches []Cache `json:"caches"` +} + +type Cache struct { + // Size of memory cache in bytes. + Size uint64 `json:"size"` + // Type of memory cache: data, instruction, or unified. + Type string `json:"type"` + // Level (distance from cpus) in a multi-level cache hierarchy. + Level int `json:"level"` } func (self *Node) FindCore(id int) (bool, int) { @@ -60,6 +71,16 @@ func (self *Node) AddThread(thread int, core int) { self.Cores[coreIdx].Threads = append(self.Cores[coreIdx].Threads, thread) } +func (self *Node) AddNodeCache(c Cache) { + self.Caches = append(self.Caches, c) +} + +func (self *Node) AddPerCoreCache(c Cache) { + for idx, _ := range self.Cores { + self.Cores[idx].Caches = append(self.Cores[idx].Caches, c) + } +} + type DiskInfo struct { // device name Name string `json:"name"` diff --git a/manager/machine.go b/manager/machine.go index 5db59dd0..baeda0b1 100644 --- a/manager/machine.go +++ b/manager/machine.go @@ -128,7 +128,7 @@ func addNode(nodes *[]info.Node, id int) (int, error) { return idx, nil } -func getTopology(cpuinfo string) ([]info.Node, int, error) { +func getTopology(sysFs sysfs.SysFs, cpuinfo string) ([]info.Node, int, error) { nodes := []info.Node{} numCores := 0 lastThread := -1 @@ -177,6 +177,29 @@ func getTopology(cpuinfo string) ([]info.Node, int, error) { if numCores < 1 { return nil, numCores, fmt.Errorf("could not detect any cores") } + for idx, node := range nodes { + caches, err := sysfs.GetCacheInfo(sysFs, node.Cores[0].Id) + if err != nil { + return nil, -1, fmt.Errorf("failed to get cache information for node %d: %v", node.Id, err) + } + numThreadsPerCore := len(node.Cores[0].Threads) + numThreadsPerNode := len(node.Cores) * numThreadsPerCore + for _, cache := range caches { + c := info.Cache{ + Size: cache.Size, + Level: cache.Level, + Type: cache.Type, + } + if cache.Cpus == numThreadsPerNode && cache.Level > 2 { + // Add a node-level cache. + nodes[idx].AddNodeCache(c) + } else if cache.Cpus == numThreadsPerCore { + // Add to each core. + nodes[idx].AddPerCoreCache(c) + } + // Ignore unknown caches. + } + } return nodes, numCores, nil } @@ -212,7 +235,7 @@ func getMachineInfo(sysFs sysfs.SysFs) (*info.MachineInfo, error) { return nil, err } - topology, numCores, err := getTopology(string(cpuinfo)) + topology, numCores, err := getTopology(sysFs, string(cpuinfo)) if err != nil { return nil, err } diff --git a/manager/topology_test.go b/manager/topology_test.go index 97a8afe4..24582941 100644 --- a/manager/topology_test.go +++ b/manager/topology_test.go @@ -20,6 +20,8 @@ import ( "testing" "github.com/google/cadvisor/info" + "github.com/google/cadvisor/utils/sysfs" + "github.com/google/cadvisor/utils/sysfs/fakesysfs" ) func TestTopology(t *testing.T) { @@ -28,7 +30,15 @@ func TestTopology(t *testing.T) { if err != nil { t.Fatalf("unable to read input test file %s", testfile) } - topology, numCores, err := getTopology(string(testcpuinfo)) + sysFs := &fakesysfs.FakeSysFs{} + c := sysfs.CacheInfo{ + Size: 32 * 1024, + Type: "unified", + Level: 1, + Cpus: 2, + } + sysFs.SetCacheInfo(c) + topology, numCores, err := getTopology(sysFs, string(testcpuinfo)) if err != nil { t.Errorf("failed to get topology for sample cpuinfo %s", string(testcpuinfo)) } @@ -40,12 +50,18 @@ func TestTopology(t *testing.T) { numNodes := 2 numCoresPerNode := 3 numThreads := 2 + cache := info.Cache{ + Size: 32 * 1024, + Type: "unified", + Level: 1, + } for i := 0; i < numNodes; i++ { node := info.Node{Id: i} // Copy over Memory from result. TODO(rjnagal): Use memory from fake. node.Memory = topology[i].Memory for j := 0; j < numCoresPerNode; j++ { core := info.Core{Id: i*numCoresPerNode + j} + core.Caches = append(core.Caches, cache) for k := 0; k < numThreads; k++ { core.Threads = append(core.Threads, k*numCoresPerNode*numNodes+core.Id) } @@ -60,13 +76,27 @@ func TestTopology(t *testing.T) { } func TestTopologyWithSimpleCpuinfo(t *testing.T) { - topology, numCores, err := getTopology("processor\t: 0\n") + sysFs := &fakesysfs.FakeSysFs{} + c := sysfs.CacheInfo{ + Size: 32 * 1024, + Type: "unified", + Level: 1, + Cpus: 1, + } + sysFs.SetCacheInfo(c) + topology, numCores, err := getTopology(sysFs, "processor\t: 0\n") if err != nil { t.Errorf("Expected cpuinfo with no topology data to succeed.") } node := info.Node{Id: 0} core := info.Core{Id: 0} core.Threads = append(core.Threads, 0) + cache := info.Cache{ + Size: 32 * 1024, + Type: "unified", + Level: 1, + } + core.Caches = append(core.Caches, cache) node.Cores = append(node.Cores, core) // Copy over Memory from result. TODO(rjnagal): Use memory from fake. node.Memory = topology[0].Memory @@ -80,7 +110,7 @@ func TestTopologyWithSimpleCpuinfo(t *testing.T) { } func TestTopologyEmptyCpuinfo(t *testing.T) { - _, _, err := getTopology("") + _, _, err := getTopology(&fakesysfs.FakeSysFs{}, "") if err == nil { t.Errorf("Expected empty cpuinfo to fail.") } diff --git a/utils/sysfs/fakesysfs/fake.go b/utils/sysfs/fakesysfs/fake.go index 5af797c0..55ab4a19 100644 --- a/utils/sysfs/fakesysfs/fake.go +++ b/utils/sysfs/fakesysfs/fake.go @@ -17,14 +17,17 @@ package fakesysfs import ( "os" "time" + + "github.com/google/cadvisor/utils/sysfs" ) // If we extend sysfs to support more interfaces, it might be worth making this a mock instead of a fake. type FileInfo struct { + EntryName string } func (self *FileInfo) Name() string { - return "sda" + return self.EntryName } func (self *FileInfo) Size() int64 { @@ -48,10 +51,12 @@ func (self *FileInfo) Sys() interface{} { } type FakeSysFs struct { - info FileInfo + info FileInfo + cache sysfs.CacheInfo } func (self *FakeSysFs) GetBlockDevices() ([]os.FileInfo, error) { + self.info.EntryName = "sda" return []os.FileInfo{&self.info}, nil } @@ -62,3 +67,16 @@ func (self *FakeSysFs) GetBlockDeviceSize(name string) (string, error) { func (self *FakeSysFs) GetBlockDeviceNumbers(name string) (string, error) { return "8:0\n", nil } + +func (self *FakeSysFs) GetCaches(id int) ([]os.FileInfo, error) { + self.info.EntryName = "index0" + return []os.FileInfo{&self.info}, nil +} + +func (self *FakeSysFs) GetCacheInfo(cpu int, cache string) (sysfs.CacheInfo, error) { + return self.cache, nil +} + +func (self *FakeSysFs) SetCacheInfo(cache sysfs.CacheInfo) { + self.cache = cache +} diff --git a/utils/sysfs/sysfs.go b/utils/sysfs/sysfs.go index 01f39b9e..d5043ab0 100644 --- a/utils/sysfs/sysfs.go +++ b/utils/sysfs/sysfs.go @@ -25,7 +25,10 @@ import ( "github.com/google/cadvisor/info" ) -const BlockDir = "/sys/block" +const ( + blockDir = "/sys/block" + cacheDir = "/sys/devices/system/cpu/cpu" +) // Abstracts the lowest level calls to sysfs. type SysFs interface { @@ -35,6 +38,11 @@ type SysFs interface { GetBlockDeviceSize(string) (string, error) // Get device major:minor number string. GetBlockDeviceNumbers(string) (string, error) + + // Get directory information for available caches accessible to given cpu. + GetCaches(id int) ([]os.FileInfo, error) + // Get information for a cache accessible from the given cpu. + GetCacheInfo(cpu int, cache string) (CacheInfo, error) } type realSysFs struct{} @@ -44,11 +52,11 @@ func NewRealSysFs() (SysFs, error) { } func (self *realSysFs) GetBlockDevices() ([]os.FileInfo, error) { - return ioutil.ReadDir(BlockDir) + return ioutil.ReadDir(blockDir) } func (self *realSysFs) GetBlockDeviceNumbers(name string) (string, error) { - dev, err := ioutil.ReadFile(path.Join(BlockDir, name, "/dev")) + dev, err := ioutil.ReadFile(path.Join(blockDir, name, "/dev")) if err != nil { return "", err } @@ -56,13 +64,59 @@ func (self *realSysFs) GetBlockDeviceNumbers(name string) (string, error) { } func (self *realSysFs) GetBlockDeviceSize(name string) (string, error) { - size, err := ioutil.ReadFile(path.Join(BlockDir, name, "/size")) + size, err := ioutil.ReadFile(path.Join(blockDir, name, "/size")) if err != nil { return "", err } return string(size), nil } +func (self *realSysFs) GetCaches(id int) ([]os.FileInfo, error) { + cpuPath := fmt.Sprintf("%s%d/cache", cacheDir, id) + return ioutil.ReadDir(cpuPath) +} + +func (self *realSysFs) GetCacheInfo(id int, name string) (CacheInfo, error) { + cachePath := fmt.Sprintf("%s%d/cache/%s", cacheDir, id, name) + out, err := ioutil.ReadFile(path.Join(cachePath, "/size")) + if err != nil { + return CacheInfo{}, err + } + var size uint64 + n, err := fmt.Sscanf(string(out), "%dK", &size) + if err != nil || n != 1 { + return CacheInfo{}, err + } + // convert to bytes + size = size * 1024 + out, err = ioutil.ReadFile(path.Join(cachePath, "/level")) + if err != nil { + return CacheInfo{}, err + } + var level int + n, err = fmt.Sscanf(string(out), "%d", &level) + if err != nil || n != 1 { + return CacheInfo{}, err + } + + out, err = ioutil.ReadFile(path.Join(cachePath, "/type")) + if err != nil { + return CacheInfo{}, err + } + cacheType := strings.TrimSpace(string(out)) + out, err = ioutil.ReadFile(path.Join(cachePath, "/shared_cpu_list")) + if err != nil { + return CacheInfo{}, err + } + cpus := strings.Split(string(out), ",") + return CacheInfo{ + Size: size, + Level: level, + Type: cacheType, + Cpus: len(cpus), + }, nil +} + // Get information about block devices present on the system. // Uses the passed in system interface to retrieve the low level OS information. func GetBlockDeviceInfo(sysfs SysFs) (map[string]info.DiskInfo, error) { @@ -106,3 +160,34 @@ func GetBlockDeviceInfo(sysfs SysFs) (map[string]info.DiskInfo, error) { } return diskMap, nil } + +type CacheInfo struct { + // size in bytes + Size uint64 + // cache type - instruction, data, unified + Type string + // distance from cpus in a multi-level hierarchy + Level int + // number of cpus that can access this cache. + Cpus int +} + +func GetCacheInfo(sysfs SysFs, id int) ([]CacheInfo, error) { + caches, err := sysfs.GetCaches(id) + if err != nil { + return nil, err + } + + info := []CacheInfo{} + for _, cache := range caches { + if !strings.HasPrefix(cache.Name(), "index") { + continue + } + cacheInfo, err := sysfs.GetCacheInfo(id, cache.Name()) + if err != nil { + return nil, err + } + info = append(info, cacheInfo) + } + return info, nil +}