diff --git a/machine/info.go b/machine/info.go index a85f0e69..d543e9c3 100644 --- a/machine/info.go +++ b/machine/info.go @@ -82,7 +82,7 @@ func Info(sysFs sysfs.SysFs, fsInfo fs.FsInfo, inHostNamespace bool) (*info.Mach return nil, err } - hugePagesInfo, err := GetHugePagesInfo(hugepagesDirectory) + hugePagesInfo, err := sysinfo.GetHugePagesInfo(sysFs, hugepagesDirectory) if err != nil { return nil, err } @@ -102,7 +102,7 @@ func Info(sysFs sysfs.SysFs, fsInfo fs.FsInfo, inHostNamespace bool) (*info.Mach klog.Errorf("Failed to get network devices: %v", err) } - topology, numCores, err := GetTopology(sysFs, string(cpuinfo)) + topology, numCores, err := GetTopology(sysFs) if err != nil { klog.Errorf("Failed to get topology information: %v", err) } diff --git a/machine/machine.go b/machine/machine.go index 40daf763..c222c6ed 100644 --- a/machine/machine.go +++ b/machine/machine.go @@ -16,7 +16,6 @@ package machine import ( - "bytes" "fmt" "io/ioutil" "os" @@ -52,6 +51,7 @@ var ( cpuBusPath = "/sys/bus/cpu/devices/" isMemoryController = regexp.MustCompile("mc[0-9]+") isDimm = regexp.MustCompile("dimm[0-9]+") + machineArch = getMachineArch() ) const maxFreqFile = "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq" @@ -213,6 +213,15 @@ func GetMachineSwapCapacity() (uint64, error) { return swapCapacity, err } +// GetTopology returns CPU topology reading information from sysfs +func GetTopology(sysFs sysfs.SysFs) ([]info.Node, int, error) { + // s390/s390x changes + if isSystemZ() { + return nil, getNumCores(), nil + } + return sysinfo.GetNodesInfo(sysFs) +} + // parseCapacity matches a Regexp in a []byte, returning the resulting value in bytes. // Assumes that the value matched by the Regexp is in KB. func parseCapacity(b []byte, r *regexp.Regexp) (uint64, error) { @@ -229,30 +238,6 @@ func parseCapacity(b []byte, r *regexp.Regexp) (uint64, error) { return m * 1024, err } -// Looks for sysfs cpu path containing core_id -// Such as: sys/bus/cpu/devices/cpu0/topology/core_id -func getCoreIdFromCpuBus(cpuBusPath string, threadId int) (int, error) { - path := filepath.Join(cpuBusPath, fmt.Sprintf("cpu%d/topology", threadId)) - file := filepath.Join(path, sysFsCPUCoreID) - - num, err := ioutil.ReadFile(file) - if err != nil { - return threadId, err - } - - coreId, err := strconv.ParseInt(string(bytes.TrimSpace(num)), 10, 32) - if err != nil { - return threadId, err - } - - if coreId < 0 { - // report threadId if found coreId < 0 - coreId = int64(threadId) - } - - return int(coreId), nil -} - // Looks for sysfs cpu path containing given CPU property, e.g. core_id or physical_package_id // and returns number of unique values of given property, exemplary usage: getting number of CPU physical cores func getUniqueCPUPropertyCount(cpuBusPath string, propertyName string) int { @@ -275,192 +260,6 @@ func getUniqueCPUPropertyCount(cpuBusPath string, propertyName string) int { return len(uniques) } -// Looks for sysfs cpu path containing node id -// Such as: /sys/bus/cpu/devices/cpu0/node%d -func getNodeIdFromCpuBus(cpuBusPath string, threadId int) (int, error) { - path := filepath.Join(cpuBusPath, fmt.Sprintf("cpu%d", threadId)) - - files, err := ioutil.ReadDir(path) - if err != nil { - return 0, err - } - - nodeId := 0 - for _, file := range files { - filename := file.Name() - - ok, val, _ := extractValue(filename, nodeBusRegExp) - if ok { - if val < 0 { - continue - } - nodeId = val - break - } - } - - return nodeId, nil -} - -// GetHugePagesInfo returns information about pre-allocated huge pages -// hugepagesDirectory should be top directory of hugepages -// Such as: /sys/kernel/mm/hugepages/ -func GetHugePagesInfo(hugepagesDirectory string) ([]info.HugePagesInfo, error) { - var hugePagesInfo []info.HugePagesInfo - files, err := ioutil.ReadDir(hugepagesDirectory) - if err != nil { - // treat as non-fatal since kernels and machine can be - // configured to disable hugepage support - return hugePagesInfo, nil - } - for _, st := range files { - nameArray := strings.Split(st.Name(), "-") - pageSizeArray := strings.Split(nameArray[1], "kB") - pageSize, err := strconv.ParseUint(string(pageSizeArray[0]), 10, 64) - if err != nil { - return hugePagesInfo, err - } - - numFile := hugepagesDirectory + st.Name() + "/nr_hugepages" - val, err := ioutil.ReadFile(numFile) - if err != nil { - return hugePagesInfo, err - } - var numPages uint64 - // we use sscanf as the file as a new-line that trips up ParseUint - // it returns the number of tokens successfully parsed, so if - // n != 1, it means we were unable to parse a number from the file - n, err := fmt.Sscanf(string(val), "%d", &numPages) - if err != nil || n != 1 { - return hugePagesInfo, fmt.Errorf("could not parse file %v contents %q", numFile, string(val)) - } - - hugePagesInfo = append(hugePagesInfo, info.HugePagesInfo{ - NumPages: numPages, - PageSize: pageSize, - }) - } - return hugePagesInfo, nil -} - -func GetTopology(sysFs sysfs.SysFs, cpuinfo string) ([]info.Node, int, error) { - nodes := []info.Node{} - - // s390/s390x changes - if true == isSystemZ() { - return nodes, getNumCores(), nil - } - - numCores := 0 - lastThread := -1 - lastCore := -1 - lastNode := -1 - for _, line := range strings.Split(cpuinfo, "\n") { - if line == "" { - continue - } - ok, val, err := extractValue(line, cpuRegExp) - if err != nil { - return nil, -1, fmt.Errorf("could not parse cpu info from %q: %v", line, err) - } - if ok { - thread := val - numCores++ - if lastThread != -1 { - // New cpu section. Save last one. - nodeIdx, err := addNode(&nodes, lastNode) - if err != nil { - return nil, -1, fmt.Errorf("failed to add node %d: %v", lastNode, err) - } - nodes[nodeIdx].AddThread(lastThread, lastCore) - lastCore = -1 - lastNode = -1 - } - lastThread = thread - - /* On Arm platform, no 'core id' and 'physical id' in '/proc/cpuinfo'. */ - /* So we search sysfs cpu path directly. */ - /* This method can also be used on other platforms, such as x86, ppc64le... */ - /* /sys/bus/cpu/devices/cpu%d contains the information of 'core_id' & 'node_id'. */ - /* Such as: /sys/bus/cpu/devices/cpu0/topology/core_id */ - /* Such as: /sys/bus/cpu/devices/cpu0/node0 */ - if isAArch64() { - val, err = getCoreIdFromCpuBus(cpuBusPath, lastThread) - if err != nil { - // Report thread id if no NUMA - val = lastThread - } - lastCore = val - - val, err = getNodeIdFromCpuBus(cpuBusPath, lastThread) - if err != nil { - // Report node 0 if no NUMA - val = 0 - } - lastNode = val - } - continue - } - - if isAArch64() { - /* On Arm platform, no 'core id' and 'physical id' in '/proc/cpuinfo'. */ - continue - } - - ok, val, err = extractValue(line, coreRegExp) - if err != nil { - return nil, -1, fmt.Errorf("could not parse core info from %q: %v", line, err) - } - if ok { - lastCore = val - continue - } - - ok, val, err = extractValue(line, nodeRegExp) - if err != nil { - return nil, -1, fmt.Errorf("could not parse node info from %q: %v", line, err) - } - if ok { - lastNode = val - continue - } - } - - nodeIdx, err := addNode(&nodes, lastNode) - if err != nil { - return nil, -1, fmt.Errorf("failed to add node %d: %v", lastNode, err) - } - nodes[nodeIdx].AddThread(lastThread, lastCore) - if numCores < 1 { - return nil, numCores, fmt.Errorf("could not detect any cores") - } - for idx, node := range nodes { - caches, err := sysinfo.GetCacheInfo(sysFs, node.Cores[0].Threads[0]) - if err != nil { - klog.Errorf("failed to get cache information for node %d: %v", node.Id, err) - continue - } - numThreadsPerCore := len(node.Cores[0].Threads) - numThreadsPerNode := len(node.Cores) * numThreadsPerCore - for _, cache := range caches { - c := info.Cache{ - Size: cache.Size, - Level: cache.Level, - Type: cache.Type, - } - if cache.Cpus == numThreadsPerNode && cache.Level > 2 { - // Add a node-level cache. - nodes[idx].AddNodeCache(c) - } else if cache.Cpus == numThreadsPerCore { - // Add to each core. - nodes[idx].AddPerCoreCache(c) - } - // Ignore unknown caches. - } - } - return nodes, numCores, nil -} - func extractValue(s string, r *regexp.Regexp) (bool, int, error) { matches := r.FindSubmatch([]byte(s)) if len(matches) == 2 { @@ -483,106 +282,39 @@ func getUniqueMatchesCount(s string, r *regexp.Regexp) int { return len(uniques) } -func findNode(nodes []info.Node, id int) (bool, int) { - for i, n := range nodes { - if n.Id == id { - return true, i - } - } - return false, -1 -} - -func addNode(nodes *[]info.Node, id int) (int, error) { - var idx int - if id == -1 { - // Some VMs don't fill topology data. Export single package. - id = 0 - } - - ok, idx := findNode(*nodes, id) - if !ok { - // New node - node := info.Node{Id: id} - // Add per-node memory information. - meminfo := fmt.Sprintf("/sys/devices/system/node/node%d/meminfo", id) - out, err := ioutil.ReadFile(meminfo) - // Ignore if per-node info is not available. - if err == nil { - m, err := parseCapacity(out, memoryCapacityRegexp) - if err != nil { - return -1, err - } - node.Memory = uint64(m) - } - // Look for per-node hugepages info using node id - // Such as: /sys/devices/system/node/node%d/hugepages - hugepagesDirectory := fmt.Sprintf("%s/node%d/hugepages/", nodePath, id) - hugePagesInfo, err := GetHugePagesInfo(hugepagesDirectory) - if err != nil { - return -1, err - } - node.HugePages = hugePagesInfo - - *nodes = append(*nodes, node) - idx = len(*nodes) - 1 - } - return idx, nil -} - -// s390/s390x changes -func getMachineArch() (string, error) { +func getMachineArch() string { uname := unix.Utsname{} err := unix.Uname(&uname) if err != nil { - return "", err + klog.Errorf("Cannot get machine architecture, err: %v", err) + return "" } - - return string(uname.Machine[:]), nil + return string(uname.Machine[:]) } -// arm32 chanes +// arm32 changes func isArm32() bool { - arch, err := getMachineArch() - if err == nil { - return strings.Contains(arch, "arm") - } - return false + return strings.Contains(machineArch, "arm") } // aarch64 changes func isAArch64() bool { - arch, err := getMachineArch() - if err == nil { - return strings.Contains(arch, "aarch64") - } - return false + return strings.Contains(machineArch, "aarch64") } // s390/s390x changes func isSystemZ() bool { - arch, err := getMachineArch() - if err == nil { - return strings.Contains(arch, "390") - } - return false + return strings.Contains(machineArch, "390") } // riscv64 changes func isRiscv64() bool { - arch, err := getMachineArch() - if err == nil { - return strings.Contains(arch, "riscv64") - } - return false + return strings.Contains(machineArch, "riscv64") } // mips64 changes func isMips64() bool { - arch, err := getMachineArch() - if err == nil { - return strings.Contains(arch, "mips64") - } - return false + return strings.Contains(machineArch, "mips64") } // s390/s390x changes diff --git a/machine/topology_test.go b/machine/topology_test.go index 65c1d0fb..db70929a 100644 --- a/machine/topology_test.go +++ b/machine/topology_test.go @@ -15,9 +15,10 @@ package machine import ( + "encoding/json" "io/ioutil" + "os" "reflect" - "runtime" "testing" info "github.com/google/cadvisor/info/v1" @@ -38,7 +39,7 @@ func TestPhysicalCores(t *testing.T) { } func TestPhysicalCoresReadingFromCpuBus(t *testing.T) { - cpuBusPath = "./testdata/" // overwriting global variable to mock sysfs + cpuBusPath = "./testdata/" // overwriting package variable to mock sysfs testfile := "./testdata/cpuinfo_arm" // mock cpuinfo without core id testcpuinfo, err := ioutil.ReadFile(testfile) @@ -50,7 +51,7 @@ func TestPhysicalCoresReadingFromCpuBus(t *testing.T) { } func TestPhysicalCoresFromWrongSysFs(t *testing.T) { - cpuBusPath = "./testdata/wrongsysfs" // overwriting global variable to mock sysfs + cpuBusPath = "./testdata/wrongsysfs" // overwriting package variable to mock sysfs testfile := "./testdata/cpuinfo_arm" // mock cpuinfo without core id testcpuinfo, err := ioutil.ReadFile(testfile) @@ -73,7 +74,7 @@ func TestSockets(t *testing.T) { } func TestSocketsReadingFromCpuBus(t *testing.T) { - cpuBusPath = "./testdata/wrongsysfs" // overwriting global variable to mock sysfs + cpuBusPath = "./testdata/wrongsysfs" // overwriting package variable to mock sysfs testfile := "./testdata/cpuinfo_arm" // mock cpuinfo without physical id testcpuinfo, err := ioutil.ReadFile(testfile) @@ -85,7 +86,7 @@ func TestSocketsReadingFromCpuBus(t *testing.T) { } func TestSocketsReadingFromWrongSysFs(t *testing.T) { - cpuBusPath = "./testdata/" // overwriting global variable to mock sysfs + cpuBusPath = "./testdata/" // overwriting package variable to mock sysfs testfile := "./testdata/cpuinfo_arm" // mock cpuinfo without physical id testcpuinfo, err := ioutil.ReadFile(testfile) @@ -97,14 +98,7 @@ func TestSocketsReadingFromWrongSysFs(t *testing.T) { } func TestTopology(t *testing.T) { - if runtime.GOARCH != "amd64" { - t.Skip("cpuinfo testdata is for amd64") - } - testfile := "./testdata/cpuinfo" - testcpuinfo, err := ioutil.ReadFile(testfile) - if err != nil { - t.Fatalf("unable to read input test file %s", testfile) - } + machineArch = "" // overwrite package variable sysFs := &fakesysfs.FakeSysFs{} c := sysfs.CacheInfo{ Size: 32 * 1024, @@ -113,14 +107,70 @@ func TestTopology(t *testing.T) { Cpus: 2, } sysFs.SetCacheInfo(c) - topology, numCores, err := GetTopology(sysFs, string(testcpuinfo)) - if err != nil { - t.Errorf("failed to get topology for sample cpuinfo %s: %v", string(testcpuinfo), err) - } - if numCores != 12 { - t.Errorf("Expected 12 cores, found %d", numCores) + nodesPaths := []string{ + "/fakeSysfs/devices/system/node/node0", + "/fakeSysfs/devices/system/node/node1", } + sysFs.SetNodesPaths(nodesPaths, nil) + + cpusPaths := map[string][]string{ + "/fakeSysfs/devices/system/node/node0": { + "/fakeSysfs/devices/system/node/node0/cpu0", + "/fakeSysfs/devices/system/node/node0/cpu1", + "/fakeSysfs/devices/system/node/node0/cpu2", + "/fakeSysfs/devices/system/node/node0/cpu6", + "/fakeSysfs/devices/system/node/node0/cpu7", + "/fakeSysfs/devices/system/node/node0/cpu8", + }, + "/fakeSysfs/devices/system/node/node1": { + "/fakeSysfs/devices/system/node/node0/cpu3", + "/fakeSysfs/devices/system/node/node0/cpu4", + "/fakeSysfs/devices/system/node/node0/cpu5", + "/fakeSysfs/devices/system/node/node0/cpu9", + "/fakeSysfs/devices/system/node/node0/cpu10", + "/fakeSysfs/devices/system/node/node0/cpu11", + }, + } + sysFs.SetCPUsPaths(cpusPaths, nil) + + coreThread := map[string]string{ + "/fakeSysfs/devices/system/node/node0/cpu0": "0", + "/fakeSysfs/devices/system/node/node0/cpu1": "1", + "/fakeSysfs/devices/system/node/node0/cpu2": "2", + "/fakeSysfs/devices/system/node/node0/cpu3": "3", + "/fakeSysfs/devices/system/node/node0/cpu4": "4", + "/fakeSysfs/devices/system/node/node0/cpu5": "5", + "/fakeSysfs/devices/system/node/node0/cpu6": "0", + "/fakeSysfs/devices/system/node/node0/cpu7": "1", + "/fakeSysfs/devices/system/node/node0/cpu8": "2", + "/fakeSysfs/devices/system/node/node0/cpu9": "3", + "/fakeSysfs/devices/system/node/node0/cpu10": "4", + "/fakeSysfs/devices/system/node/node0/cpu11": "5", + } + sysFs.SetCoreThreads(coreThread, nil) + + memTotal := "MemTotal: 32817192 kB" + sysFs.SetMemory(memTotal, nil) + + hugePages := []os.FileInfo{ + &fakesysfs.FileInfo{EntryName: "hugepages-2048kB"}, + &fakesysfs.FileInfo{EntryName: "hugepages-1048576kB"}, + } + sysFs.SetHugePages(hugePages, nil) + + hugePageNr := map[string]string{ + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages": "1", + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages": "1", + "/fakeSysfs/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages": "1", + "/fakeSysfs/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages": "1", + } + sysFs.SetHugePagesNr(hugePageNr, nil) + + topology, numCores, err := GetTopology(sysFs) + assert.Nil(t, err) + assert.Equal(t, 12, numCores) + expected_topology := []info.Node{} numNodes := 2 numCoresPerNode := 3 @@ -147,105 +197,93 @@ func TestTopology(t *testing.T) { expected_topology = append(expected_topology, node) } - if !reflect.DeepEqual(topology, expected_topology) { - t.Errorf("Expected topology %+v, got %+v", expected_topology, topology) - } + assert.NotNil(t, reflect.DeepEqual(topology, expected_topology)) } -func TestTopologyWithSimpleCpuinfo(t *testing.T) { - if isSystemZ() { - t.Skip("systemZ has no topology info") - } +func TestTopologyEmptySysFs(t *testing.T) { + machineArch = "" // overwrite package variable + _, _, err := GetTopology(&fakesysfs.FakeSysFs{}) + assert.NotNil(t, err) +} + +func TestTopologyWithNodesWithoutCPU(t *testing.T) { + machineArch = "" // overwrite package variable sysFs := &fakesysfs.FakeSysFs{} - c := sysfs.CacheInfo{ - Size: 32 * 1024, - Type: "unified", - Level: 1, - Cpus: 1, + nodesPaths := []string{ + "/fakeSysfs/devices/system/node/node0", + "/fakeSysfs/devices/system/node/node1", } - sysFs.SetCacheInfo(c) - topology, numCores, err := GetTopology(sysFs, "processor\t: 0\n") - if err != nil { - t.Errorf("Expected cpuinfo with no topology data to succeed.") + sysFs.SetNodesPaths(nodesPaths, nil) + + memTotal := "MemTotal: 32817192 kB" + sysFs.SetMemory(memTotal, nil) + + hugePages := []os.FileInfo{ + &fakesysfs.FileInfo{EntryName: "hugepages-2048kB"}, + &fakesysfs.FileInfo{EntryName: "hugepages-1048576kB"}, } - node := info.Node{Id: 0} - core := info.Core{Id: 0} - core.Threads = append(core.Threads, 0) - cache := info.Cache{ - Size: 32 * 1024, - Type: "unified", - Level: 1, - } - core.Caches = append(core.Caches, cache) - node.Cores = append(node.Cores, core) - // Copy over Memory from result. TODO(rjnagal): Use memory from fake. - node.Memory = topology[0].Memory - // Copy over HugePagesInfo from result. TODO(ohsewon): Use HugePagesInfo from fake. - node.HugePages = topology[0].HugePages - expected := []info.Node{node} - if !reflect.DeepEqual(topology, expected) { - t.Errorf("Expected topology %+v, got %+v", expected, topology) - } - if numCores != 1 { - t.Errorf("Expected 1 core, found %d", numCores) + sysFs.SetHugePages(hugePages, nil) + + hugePageNr := map[string]string{ + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages": "1", + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages": "1", + "/fakeSysfs/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages": "1", + "/fakeSysfs/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages": "1", } + sysFs.SetHugePagesNr(hugePageNr, nil) + + topology, numCores, err := GetTopology(sysFs) + + assert.Nil(t, err) + assert.Equal(t, 0, numCores) + + topologyJSON, err := json.Marshal(topology) + assert.Nil(t, err) + + expectedTopology := `[ + { + "caches": null, + "cores": null, + "hugepages": [ + { + "num_pages": 1, + "page_size": 2048 + }, + { + "num_pages": 1, + "page_size": 1048576 + } + ], + "memory": 33604804608, + "node_id": 0 + }, + { + "caches": null, + "cores": null, + "hugepages": [ + { + "num_pages": 1, + "page_size": 2048 + }, + { + "num_pages": 1, + "page_size": 1048576 + } + ], + "memory": 33604804608, + "node_id": 1 + } + ] + ` + assert.JSONEq(t, expectedTopology, string(topologyJSON)) } -func TestTopologyEmptyCpuinfo(t *testing.T) { - if isSystemZ() { - t.Skip("systemZ has no topology info") - } - _, _, err := GetTopology(&fakesysfs.FakeSysFs{}, "") - if err == nil { - t.Errorf("Expected empty cpuinfo to fail.") - } -} - -func TestTopologyCoreId(t *testing.T) { - val, _ := getCoreIdFromCpuBus("./testdata", 0) - if val != 0 { - t.Errorf("Expected core 0, found %d", val) - } - - val, _ = getCoreIdFromCpuBus("./testdata", 9999) - if val != 8888 { - t.Errorf("Expected core 8888, found %d", val) - } -} - -func TestTopologyNodeId(t *testing.T) { - val, _ := getNodeIdFromCpuBus("./testdata", 0) - if val != 0 { - t.Errorf("Expected core 0, found %d", val) - } - - val, _ = getNodeIdFromCpuBus("./testdata", 9999) - if val != 1234 { - t.Errorf("Expected core 1234 , found %d", val) - } -} - -func TestGetHugePagesInfo(t *testing.T) { - testPath := "./testdata/hugepages/" - expected := []info.HugePagesInfo{ - { - NumPages: 1, - PageSize: 1048576, - }, - { - NumPages: 2, - PageSize: 2048, - }, - } - - val, err := GetHugePagesInfo(testPath) - if err != nil { - t.Errorf("Failed to GetHugePagesInfo() for sample path %s: %v", testPath, err) - } - - if !reflect.DeepEqual(expected, val) { - t.Errorf("Expected HugePagesInfo %+v, got %+v", expected, val) - } +func TestTopologyOnSystemZ(t *testing.T) { + machineArch = "s390" // overwrite package variable + nodes, cores, err := GetTopology(&fakesysfs.FakeSysFs{}) + assert.Nil(t, err) + assert.Nil(t, nodes) + assert.NotNil(t, cores) } func TestMemoryInfo(t *testing.T) { diff --git a/utils/sysfs/fakesysfs/fake.go b/utils/sysfs/fakesysfs/fake.go index 963fc61e..fde01199 100644 --- a/utils/sysfs/fakesysfs/fake.go +++ b/utils/sysfs/fakesysfs/fake.go @@ -15,6 +15,7 @@ package fakesysfs import ( + "fmt" "os" "time" @@ -53,6 +54,49 @@ func (self *FileInfo) Sys() interface{} { type FakeSysFs struct { info FileInfo cache sysfs.CacheInfo + + nodesPaths []string + nodePathErr error + + cpusPaths map[string][]string + cpuPathErr error + + coreThread map[string]string + coreIDErr error + + memTotal string + memErr error + + hugePages []os.FileInfo + hugePagesErr error + + hugePagesNr map[string]string + hugePagesNrErr error +} + +func (self *FakeSysFs) GetNodesPaths() ([]string, error) { + return self.nodesPaths, self.nodePathErr +} + +func (self *FakeSysFs) GetCPUsPaths(nodePath string) ([]string, error) { + return self.cpusPaths[nodePath], self.cpuPathErr +} + +func (self *FakeSysFs) GetCoreID(coreIDPath string) (string, error) { + return self.coreThread[coreIDPath], self.coreIDErr +} + +func (self *FakeSysFs) GetMemInfo(nodePath string) (string, error) { + return self.memTotal, self.memErr +} + +func (self *FakeSysFs) GetHugePagesInfo(hugepagesDirectory string) ([]os.FileInfo, error) { + return self.hugePages, self.hugePagesErr +} + +func (self *FakeSysFs) GetHugePagesNr(hugepagesDirectory string, hugePageName string) (string, error) { + hugePageFile := fmt.Sprintf("%s%s/%s", hugepagesDirectory, hugePageName, sysfs.HugePagesNrFile) + return self.hugePagesNr[hugePageFile], self.hugePagesNrErr } func (self *FakeSysFs) GetBlockDevices() ([]os.FileInfo, error) { @@ -105,6 +149,36 @@ func (self *FakeSysFs) SetCacheInfo(cache sysfs.CacheInfo) { self.cache = cache } +func (self *FakeSysFs) SetNodesPaths(paths []string, err error) { + self.nodesPaths = paths + self.nodePathErr = err +} + +func (self *FakeSysFs) SetCPUsPaths(paths map[string][]string, err error) { + self.cpusPaths = paths + self.cpuPathErr = err +} + +func (self *FakeSysFs) SetCoreThreads(coreThread map[string]string, err error) { + self.coreThread = coreThread + self.coreIDErr = err +} + +func (self *FakeSysFs) SetMemory(memTotal string, err error) { + self.memTotal = memTotal + self.memErr = err +} + +func (self *FakeSysFs) SetHugePages(hugePages []os.FileInfo, err error) { + self.hugePages = hugePages + self.hugePagesErr = err +} + +func (self *FakeSysFs) SetHugePagesNr(hugePagesNr map[string]string, err error) { + self.hugePagesNr = hugePagesNr + self.hugePagesNrErr = err +} + func (self *FakeSysFs) SetEntryName(name string) { self.info.EntryName = name } diff --git a/utils/sysfs/sysfs.go b/utils/sysfs/sysfs.go index 1132e5b5..38314e5f 100644 --- a/utils/sysfs/sysfs.go +++ b/utils/sysfs/sysfs.go @@ -19,6 +19,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "strconv" "strings" ) @@ -30,6 +31,20 @@ const ( dmiDir = "/sys/class/dmi" ppcDevTree = "/proc/device-tree" s390xDevTree = "/etc" // s390/s390x changes + + hugePagesDirName = "hugepages" + coreIDFilePath = "/topology/core_id" + meminfoFile = "meminfo" + + cpuDirPattern = "cpu*[0-9]" + nodeDirPattern = "node*[0-9]" + + //HugePagesNrFile name of nr_hugepages file in sysfs + HugePagesNrFile = "nr_hugepages" +) + +var ( + nodeDir = "/sys/devices/system/node/" ) type CacheInfo struct { @@ -45,6 +60,18 @@ type CacheInfo struct { // Abstracts the lowest level calls to sysfs. type SysFs interface { + // Get NUMA nodes paths + GetNodesPaths() ([]string, error) + // Get paths to CPU assigned for specified NUMA node + GetCPUsPaths(nodePath string) ([]string, error) + // Get physical core id for specified CPU + GetCoreID(coreIDFilePath string) (string, error) + // Get total memory for specified NUMA node + GetMemInfo(nodeDir string) (string, error) + // Get hugepages from specified directory + GetHugePagesInfo(hugePagesDirectory string) ([]os.FileInfo, error) + // Get hugepage_nr from specified directory + GetHugePagesNr(hugePagesDirectory string, hugePageName string) (string, error) // Get directory information for available block devices. GetBlockDevices() ([]os.FileInfo, error) // Get Size of a given block device. @@ -74,6 +101,47 @@ func NewRealSysFs() SysFs { return &realSysFs{} } +func (self *realSysFs) GetNodesPaths() ([]string, error) { + pathPattern := fmt.Sprintf("%s%s", nodeDir, nodeDirPattern) + return filepath.Glob(pathPattern) +} + +func (self *realSysFs) GetCPUsPaths(nodePath string) ([]string, error) { + pathPattern := fmt.Sprintf("%s/%s", nodePath, cpuDirPattern) + return filepath.Glob(pathPattern) +} + +func (self *realSysFs) GetCoreID(cpuPath string) (string, error) { + coreIDFilePath := fmt.Sprintf("%s%s", cpuPath, coreIDFilePath) + coreID, err := ioutil.ReadFile(coreIDFilePath) + if err != nil { + return "", err + } + return strings.TrimSpace(string(coreID)), err +} + +func (self *realSysFs) GetMemInfo(nodePath string) (string, error) { + meminfoPath := fmt.Sprintf("%s/%s", nodePath, meminfoFile) + meminfo, err := ioutil.ReadFile(meminfoPath) + if err != nil { + return "", err + } + return strings.TrimSpace(string(meminfo)), err +} + +func (self *realSysFs) GetHugePagesInfo(hugePagesDirectory string) ([]os.FileInfo, error) { + return ioutil.ReadDir(hugePagesDirectory) +} + +func (self *realSysFs) GetHugePagesNr(hugepagesDirectory string, hugePageName string) (string, error) { + hugePageFilePath := fmt.Sprintf("%s%s/%s", hugepagesDirectory, hugePageName, HugePagesNrFile) + hugePageFile, err := ioutil.ReadFile(hugePageFilePath) + if err != nil { + return "", err + } + return strings.TrimSpace(string(hugePageFile)), err +} + func (self *realSysFs) GetBlockDevices() ([]os.FileInfo, error) { return ioutil.ReadDir(blockDir) } diff --git a/utils/sysfs/sysfs_test.go b/utils/sysfs/sysfs_test.go new file mode 100644 index 00000000..7bfa7d95 --- /dev/null +++ b/utils/sysfs/sysfs_test.go @@ -0,0 +1,124 @@ +// Copyright 2020 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysfs + +import ( + "github.com/stretchr/testify/assert" + "os" + "strconv" + "testing" +) + +func TestGetNodes(t *testing.T) { + //overwrite global variable + nodeDir = "./testdata/" + + sysFs := NewRealSysFs() + nodesDirs, err := sysFs.GetNodesPaths() + assert.Nil(t, err) + assert.Equal(t, 2, len(nodesDirs)) + assert.Contains(t, nodesDirs, "testdata/node0") + assert.Contains(t, nodesDirs, "testdata/node1") +} + +func TestGetNodesWithNonExistingDir(t *testing.T) { + //overwrite global variable + nodeDir = "./testdata/NonExistingDir/" + + sysFs := NewRealSysFs() + nodesDirs, err := sysFs.GetNodesPaths() + assert.Nil(t, err) + assert.Equal(t, 0, len(nodesDirs)) +} + +func TestGetCPUsPaths(t *testing.T) { + sysFs := NewRealSysFs() + cpuDirs, err := sysFs.GetCPUsPaths("./testdata/node0") + assert.Nil(t, err) + assert.Equal(t, 2, len(cpuDirs)) + assert.Contains(t, cpuDirs, "testdata/node0/cpu0") + assert.Contains(t, cpuDirs, "testdata/node0/cpu1") +} + +func TestGetCPUsPathsFromNodeWithoutCPU(t *testing.T) { + sysFs := NewRealSysFs() + cpuDirs, err := sysFs.GetCPUsPaths("./testdata/node1") + assert.Nil(t, err) + assert.Equal(t, 0, len(cpuDirs)) +} + +func TestGetCoreID(t *testing.T) { + sysFs := NewRealSysFs() + rawCoreID, err := sysFs.GetCoreID("./testdata/node0/cpu0") + assert.Nil(t, err) + + coreID, err := strconv.Atoi(rawCoreID) + assert.Nil(t, err) + assert.Equal(t, 0, coreID) +} + +func TestGetCoreIDWhenFileIsMissing(t *testing.T) { + sysFs := NewRealSysFs() + rawCoreID, err := sysFs.GetCoreID("./testdata/node0/cpu1") + assert.NotNil(t, err) + assert.Equal(t, "", rawCoreID) +} + +func TestGetMemInfo(t *testing.T) { + sysFs := NewRealSysFs() + memInfo, err := sysFs.GetMemInfo("./testdata/node0") + assert.Nil(t, err) + assert.Equal(t, "Node 0 MemTotal: 32817192 kB", memInfo) +} + +func TestGetMemInfoWhenFileIsMissing(t *testing.T) { + sysFs := NewRealSysFs() + memInfo, err := sysFs.GetMemInfo("./testdata/node1") + assert.NotNil(t, err) + assert.Equal(t, "", memInfo) +} + +func TestGetHugePagesInfo(t *testing.T) { + sysFs := NewRealSysFs() + hugePages, err := sysFs.GetHugePagesInfo("./testdata/node0/hugepages") + assert.Nil(t, err) + assert.Equal(t, 2, len(hugePages)) + assert.Equal(t, "hugepages-1048576kB", hugePages[0].Name()) + assert.Equal(t, "hugepages-2048kB", hugePages[1].Name()) +} + +func TestGetHugePagesInfoWhenDirIsMissing(t *testing.T) { + sysFs := NewRealSysFs() + hugePages, err := sysFs.GetHugePagesInfo("./testdata/node1/hugepages") + assert.NotNil(t, err) + assert.Equal(t, []os.FileInfo([]os.FileInfo(nil)), hugePages) +} + +func TestGetHugePagesNr(t *testing.T) { + sysFs := NewRealSysFs() + rawHugePageNr, err := sysFs.GetHugePagesNr("./testdata/node0/hugepages/", "hugepages-1048576kB") + assert.Nil(t, err) + + hugePageNr, err := strconv.Atoi(rawHugePageNr) + assert.Nil(t, err) + assert.Equal(t, 1, hugePageNr) +} + +func TestGetHugePagesNrWhenFileIsMissing(t *testing.T) { + sysFs := NewRealSysFs() + rawHugePageNr, err := sysFs.GetHugePagesNr("./testdata/node1/hugepages/", "hugepages-1048576kB") + assert.NotNil(t, err) + assert.Equal(t, "", rawHugePageNr) +} diff --git a/utils/sysfs/testdata/node0/cpu0/topology/core_id b/utils/sysfs/testdata/node0/cpu0/topology/core_id new file mode 100644 index 00000000..573541ac --- /dev/null +++ b/utils/sysfs/testdata/node0/cpu0/topology/core_id @@ -0,0 +1 @@ +0 diff --git a/machine/testdata/cpu0/node0/null b/utils/sysfs/testdata/node0/cpu1/topology/.gitkeep similarity index 100% rename from machine/testdata/cpu0/node0/null rename to utils/sysfs/testdata/node0/cpu1/topology/.gitkeep diff --git a/machine/testdata/hugepages/hugepages-1048576kB/nr_hugepages b/utils/sysfs/testdata/node0/hugepages/hugepages-1048576kB/nr_hugepages similarity index 100% rename from machine/testdata/hugepages/hugepages-1048576kB/nr_hugepages rename to utils/sysfs/testdata/node0/hugepages/hugepages-1048576kB/nr_hugepages diff --git a/machine/testdata/hugepages/hugepages-2048kB/nr_hugepages b/utils/sysfs/testdata/node0/hugepages/hugepages-2048kB/nr_hugepages similarity index 100% rename from machine/testdata/hugepages/hugepages-2048kB/nr_hugepages rename to utils/sysfs/testdata/node0/hugepages/hugepages-2048kB/nr_hugepages diff --git a/utils/sysfs/testdata/node0/meminfo b/utils/sysfs/testdata/node0/meminfo new file mode 100644 index 00000000..dc466798 --- /dev/null +++ b/utils/sysfs/testdata/node0/meminfo @@ -0,0 +1 @@ +Node 0 MemTotal: 32817192 kB diff --git a/machine/testdata/cpu9999/node1234/null b/utils/sysfs/testdata/node1/.gitkeep similarity index 100% rename from machine/testdata/cpu9999/node1234/null rename to utils/sysfs/testdata/node1/.gitkeep diff --git a/utils/sysinfo/sysinfo.go b/utils/sysinfo/sysinfo.go index 7dd14cf4..25470d67 100644 --- a/utils/sysinfo/sysinfo.go +++ b/utils/sysinfo/sysinfo.go @@ -22,9 +22,22 @@ import ( info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/utils/sysfs" + "github.com/google/go-cmp/cmp" + + "k8s.io/klog" ) -var schedulerRegExp = regexp.MustCompile(`.*\[(.*)\].*`) +var ( + schedulerRegExp = regexp.MustCompile(`.*\[(.*)\].*`) + nodeDirRegExp = regexp.MustCompile("node/node(\\d*)") + cpuDirRegExp = regexp.MustCompile("/cpu(\\d*)") + memoryCapacityRegexp = regexp.MustCompile(`MemTotal:\s*([0-9]+) kB`) +) + +const ( + cacheLevel2 = 2 + hugepagesDir = "hugepages/" +) // Get information about block devices present on the system. // Uses the passed in system interface to retrieve the low level OS information. @@ -133,6 +146,199 @@ func GetNetworkDevices(sysfs sysfs.SysFs) ([]info.NetInfo, error) { return netDevices, nil } +// GetHugePagesInfo returns information about pre-allocated huge pages +// hugepagesDirectory should be top directory of hugepages +// Such as: /sys/kernel/mm/hugepages/ +func GetHugePagesInfo(sysFs sysfs.SysFs, hugepagesDirectory string) ([]info.HugePagesInfo, error) { + var hugePagesInfo []info.HugePagesInfo + files, err := sysFs.GetHugePagesInfo(hugepagesDirectory) + if err != nil { + // treat as non-fatal since kernels and machine can be + // configured to disable hugepage support + return hugePagesInfo, nil + } + + for _, st := range files { + nameArray := strings.Split(st.Name(), "-") + pageSizeArray := strings.Split(nameArray[1], "kB") + pageSize, err := strconv.ParseUint(string(pageSizeArray[0]), 10, 64) + if err != nil { + return hugePagesInfo, err + } + + val, err := sysFs.GetHugePagesNr(hugepagesDirectory, st.Name()) + if err != nil { + return hugePagesInfo, err + } + var numPages uint64 + // we use sscanf as the file as a new-line that trips up ParseUint + // it returns the number of tokens successfully parsed, so if + // n != 1, it means we were unable to parse a number from the file + n, err := fmt.Sscanf(string(val), "%d", &numPages) + if err != nil || n != 1 { + return hugePagesInfo, fmt.Errorf("could not parse file nr_hugepage for %s, contents %q", st.Name(), string(val)) + } + + hugePagesInfo = append(hugePagesInfo, info.HugePagesInfo{ + NumPages: numPages, + PageSize: pageSize, + }) + } + return hugePagesInfo, nil +} + +// GetNodesInfo returns information about NUMA nodes and their topology +func GetNodesInfo(sysFs sysfs.SysFs) ([]info.Node, int, error) { + nodes := []info.Node{} + allLogicalCoresCount := 0 + + nodesDirs, err := sysFs.GetNodesPaths() + if err != nil || len(nodesDirs) == 0 { + if len(nodesDirs) == 0 && err == nil { + //sysFs.GetNodesPaths uses filePath.Glob which does not return any error if pattern does not match anything + err = fmt.Errorf("Any path to specific node is not found") + } + return nil, 0, err + } + + for _, nodeDir := range nodesDirs { + id, err := getMatchedInt(nodeDirRegExp, nodeDir) + node := info.Node{Id: id} + + cores, logicalCoreCount, err := getCoresInfo(sysFs, nodeDir) + if err != nil { + return nil, 0, err + } + node.Cores = cores + + allLogicalCoresCount += logicalCoreCount + err = addCacheInfo(sysFs, &node) + if err != nil { + return nil, 0, err + } + + node.Memory, err = getNodeMemInfo(sysFs, nodeDir) + if err != nil { + return nil, 0, err + } + + hugepagesDirectory := fmt.Sprintf("%s/%s", nodeDir, hugepagesDir) + node.HugePages, err = GetHugePagesInfo(sysFs, hugepagesDirectory) + if err != nil { + return nil, 0, err + } + + nodes = append(nodes, node) + } + return nodes, allLogicalCoresCount, err +} + +// addCacheInfo adds information about cache for NUMA node +func addCacheInfo(sysFs sysfs.SysFs, node *info.Node) error { + for coreID, core := range node.Cores { + threadID := core.Threads[0] //get any thread for core + caches, err := GetCacheInfo(sysFs, threadID) + if err != nil { + return err + } + + numThreadsPerCore := len(core.Threads) + numThreadsPerNode := len(node.Cores) * numThreadsPerCore + + for _, cache := range caches { + c := info.Cache{ + Size: cache.Size, + Level: cache.Level, + Type: cache.Type, + } + if cache.Cpus == numThreadsPerNode && cache.Level > cacheLevel2 { + // Add a node-level cache. + cacheFound := false + for _, nodeCache := range node.Caches { + if cmp.Equal(nodeCache, c) { + cacheFound = true + } + } + if !cacheFound { + node.Caches = append(node.Caches, c) + } + } else if cache.Cpus == numThreadsPerCore { + // Add core level cache + node.Cores[coreID].Caches = append(node.Cores[coreID].Caches, c) + } + // Ignore unknown caches. + } + } + return nil +} + +// getNodeMemInfo returns information about total memory for NUMA node +func getNodeMemInfo(sysFs sysfs.SysFs, nodeDir string) (uint64, error) { + rawMem, err := sysFs.GetMemInfo(nodeDir) + if err != nil { + //Ignore if per-node info is not available. + klog.Warningf("Found node without memory information, nodeDir: %s", nodeDir) + return 0, nil + } + matches := memoryCapacityRegexp.FindStringSubmatch(rawMem) + if len(matches) != 2 { + return 0, fmt.Errorf("failed to match regexp in output: %q", string(rawMem)) + } + memory, err := strconv.ParseUint(matches[1], 10, 64) + if err != nil { + return 0, err + } + memory = memory * 1024 // Convert to bytes + return uint64(memory), nil +} + +// getCoresInfo retruns infromation about physical and logical cores assigned to NUMA node +func getCoresInfo(sysFs sysfs.SysFs, nodeDir string) ([]info.Core, int, error) { + cpuDirs, err := sysFs.GetCPUsPaths(nodeDir) + if err != nil || len(cpuDirs) == 0 { + klog.Warningf("Found node without any CPU, nodeDir: %s, number of cpuDirs %d, err: %v", nodeDir, len(cpuDirs), err) + return nil, 0, nil + } + + cores := make([]info.Core, 0, len(cpuDirs)) + for _, cpuDir := range cpuDirs { + cpuID, err := getMatchedInt(cpuDirRegExp, cpuDir) + if err != nil { + return nil, 0, fmt.Errorf("Unexpected format of CPU directory, cpuDirRegExp %s, cpuDir: %s", cpuDirRegExp, cpuDir) + } + + rawPhysicalID, err := sysFs.GetCoreID(cpuDir) + if err != nil { + return nil, 0, err + } + physicalID, err := strconv.Atoi(rawPhysicalID) + if err != nil { + return nil, 0, err + } + + coreIDx := -1 + for id, core := range cores { + if core.Id == physicalID { + coreIDx = id + } + } + if coreIDx == -1 { + cores = append(cores, info.Core{}) + coreIDx = len(cores) - 1 + } + desiredCore := &cores[coreIDx] + + desiredCore.Id = physicalID + if len(desiredCore.Threads) == 0 { + desiredCore.Threads = []int{cpuID} + } else { + desiredCore.Threads = append(desiredCore.Threads, cpuID) + } + } + return cores, len(cpuDirs), nil +} + +// GetCacheInfo return information about a cache accessible from the given cpu thread func GetCacheInfo(sysFs sysfs.SysFs, id int) ([]sysfs.CacheInfo, error) { caches, err := sysFs.GetCaches(id) if err != nil { @@ -201,3 +407,15 @@ func getNetworkStats(name string, sysFs sysfs.SysFs) (info.InterfaceStats, error func GetSystemUUID(sysFs sysfs.SysFs) (string, error) { return sysFs.GetSystemUUID() } + +func getMatchedInt(rgx *regexp.Regexp, str string) (int, error) { + matches := rgx.FindStringSubmatch(str) + if len(matches) != 2 { + return 0, fmt.Errorf("failed to match regexp, str: %s", str) + } + valInt, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, err + } + return valInt, nil +} diff --git a/utils/sysinfo/sysinfo_test.go b/utils/sysinfo/sysinfo_test.go index 97df16b9..63de9d0c 100644 --- a/utils/sysinfo/sysinfo_test.go +++ b/utils/sysinfo/sysinfo_test.go @@ -15,13 +15,415 @@ package sysinfo import ( + "encoding/json" + "fmt" + "os" "testing" info "github.com/google/cadvisor/info/v1" "github.com/google/cadvisor/utils/sysfs" "github.com/google/cadvisor/utils/sysfs/fakesysfs" + "github.com/stretchr/testify/assert" ) +func TestGetHugePagesInfo(t *testing.T) { + fakeSys := fakesysfs.FakeSysFs{} + hugePages := []os.FileInfo{ + &fakesysfs.FileInfo{EntryName: "hugepages-2048kB"}, + &fakesysfs.FileInfo{EntryName: "hugepages-1048576kB"}, + } + fakeSys.SetHugePages(hugePages, nil) + + hugePageNr := map[string]string{ + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages": "1", + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages": "1", + } + fakeSys.SetHugePagesNr(hugePageNr, nil) + + hugePagesInfo, err := GetHugePagesInfo(&fakeSys, "/fakeSysfs/devices/system/node/node0/hugepages/") + assert.Nil(t, err) + assert.Equal(t, 2, len(hugePagesInfo)) +} + +func TestGetHugePagesInfoWithHugePagesDirectory(t *testing.T) { + fakeSys := fakesysfs.FakeSysFs{} + hugePagesInfo, err := GetHugePagesInfo(&fakeSys, "/fakeSysfs/devices/system/node/node0/hugepages/") + assert.Nil(t, err) + assert.Equal(t, 0, len(hugePagesInfo)) +} + +func TestGetHugePagesInfoWithWrongDirName(t *testing.T) { + fakeSys := fakesysfs.FakeSysFs{} + hugePages := []os.FileInfo{ + &fakesysfs.FileInfo{EntryName: "hugepages-abckB"}, + } + fakeSys.SetHugePages(hugePages, nil) + + hugePagesInfo, err := GetHugePagesInfo(&fakeSys, "/fakeSysfs/devices/system/node/node0/hugepages/") + assert.NotNil(t, err) + assert.Equal(t, 0, len(hugePagesInfo)) +} + +func TestGetHugePagesInfoWithReadingNrHugePagesError(t *testing.T) { + fakeSys := fakesysfs.FakeSysFs{} + hugePages := []os.FileInfo{ + &fakesysfs.FileInfo{EntryName: "hugepages-2048kB"}, + &fakesysfs.FileInfo{EntryName: "hugepages-1048576kB"}, + } + fakeSys.SetHugePages(hugePages, nil) + + hugePageNr := map[string]string{ + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages": "1", + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages": "1", + } + fakeSys.SetHugePagesNr(hugePageNr, fmt.Errorf("Error in reading nr_hugepages")) + + hugePagesInfo, err := GetHugePagesInfo(&fakeSys, "/fakeSysfs/devices/system/node/node0/hugepages/") + assert.NotNil(t, err) + assert.Equal(t, 0, len(hugePagesInfo)) +} + +func TestGetHugePagesInfoWithWrongNrHugePageValue(t *testing.T) { + fakeSys := fakesysfs.FakeSysFs{} + hugePages := []os.FileInfo{ + &fakesysfs.FileInfo{EntryName: "hugepages-2048kB"}, + &fakesysfs.FileInfo{EntryName: "hugepages-1048576kB"}, + } + fakeSys.SetHugePages(hugePages, nil) + + hugePageNr := map[string]string{ + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages": "*****", + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages": "1", + } + fakeSys.SetHugePagesNr(hugePageNr, nil) + + hugePagesInfo, err := GetHugePagesInfo(&fakeSys, "/fakeSysfs/devices/system/node/node0/hugepages/") + assert.NotNil(t, err) + assert.Equal(t, 0, len(hugePagesInfo)) +} + +func TestGetNodesInfo(t *testing.T) { + fakeSys := &fakesysfs.FakeSysFs{} + c := sysfs.CacheInfo{ + Size: 32 * 1024, + Type: "unified", + Level: 3, + Cpus: 2, + } + fakeSys.SetCacheInfo(c) + + nodesPaths := []string{ + "/fakeSysfs/devices/system/node/node0", + "/fakeSysfs/devices/system/node/node1", + } + fakeSys.SetNodesPaths(nodesPaths, nil) + + cpusPaths := map[string][]string{ + "/fakeSysfs/devices/system/node/node0": { + "/fakeSysfs/devices/system/node/node0/cpu0", + "/fakeSysfs/devices/system/node/node0/cpu1", + }, + "/fakeSysfs/devices/system/node/node1": { + "/fakeSysfs/devices/system/node/node0/cpu2", + "/fakeSysfs/devices/system/node/node0/cpu3", + }, + } + fakeSys.SetCPUsPaths(cpusPaths, nil) + + coreThread := map[string]string{ + "/fakeSysfs/devices/system/node/node0/cpu0": "0", + "/fakeSysfs/devices/system/node/node0/cpu1": "0", + "/fakeSysfs/devices/system/node/node0/cpu2": "1", + "/fakeSysfs/devices/system/node/node0/cpu3": "1", + } + fakeSys.SetCoreThreads(coreThread, nil) + + memTotal := "MemTotal: 32817192 kB" + fakeSys.SetMemory(memTotal, nil) + + hugePages := []os.FileInfo{ + &fakesysfs.FileInfo{EntryName: "hugepages-2048kB"}, + } + fakeSys.SetHugePages(hugePages, nil) + + hugePageNr := map[string]string{ + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages": "1", + "/fakeSysfs/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages": "1", + } + fakeSys.SetHugePagesNr(hugePageNr, nil) + + nodes, cores, err := GetNodesInfo(fakeSys) + assert.Nil(t, err) + assert.Equal(t, 2, len(nodes)) + assert.Equal(t, 4, cores) + + nodesJSON, err := json.Marshal(nodes) + assert.Nil(t, err) + expectedNodes := ` + [ + { + "node_id": 0, + "memory": 33604804608, + "hugepages": [ + { + "page_size": 2048, + "num_pages": 1 + } + ], + "cores": [ + { + "core_id": 0, + "thread_ids": [ + 0, + 1 + ], + "caches": null + } + ], + "caches": [ + { + "size": 32768, + "type": "unified", + "level": 3 + } + ] + }, + { + "node_id": 1, + "memory": 33604804608, + "hugepages": [ + { + "page_size": 2048, + "num_pages": 1 + } + ], + "cores": [ + { + "core_id": 1, + "thread_ids": [ + 2, + 3 + ], + "caches": null + } + ], + "caches": [ + { + "size": 32768, + "type": "unified", + "level": 3 + } + ] + } + ] + ` + assert.JSONEq(t, expectedNodes, string(nodesJSON)) +} + +func TestGetNodesWithoutMemoryInfo(t *testing.T) { + fakeSys := &fakesysfs.FakeSysFs{} + c := sysfs.CacheInfo{ + Size: 32 * 1024, + Type: "unified", + Level: 3, + Cpus: 2, + } + fakeSys.SetCacheInfo(c) + + nodesPaths := []string{ + "/fakeSysfs/devices/system/node/node0", + "/fakeSysfs/devices/system/node/node1", + } + fakeSys.SetNodesPaths(nodesPaths, nil) + + cpusPaths := map[string][]string{ + "/fakeSysfs/devices/system/node/node0": { + "/fakeSysfs/devices/system/node/node0/cpu0", + "/fakeSysfs/devices/system/node/node0/cpu1", + }, + "/fakeSysfs/devices/system/node/node1": { + "/fakeSysfs/devices/system/node/node0/cpu2", + "/fakeSysfs/devices/system/node/node0/cpu3", + }, + } + fakeSys.SetCPUsPaths(cpusPaths, nil) + + coreThread := map[string]string{ + "/fakeSysfs/devices/system/node/node0/cpu0": "0", + "/fakeSysfs/devices/system/node/node0/cpu1": "0", + "/fakeSysfs/devices/system/node/node0/cpu2": "1", + "/fakeSysfs/devices/system/node/node0/cpu3": "1", + } + fakeSys.SetCoreThreads(coreThread, nil) + + hugePages := []os.FileInfo{ + &fakesysfs.FileInfo{EntryName: "hugepages-2048kB"}, + } + fakeSys.SetHugePages(hugePages, nil) + + hugePageNr := map[string]string{ + "/fakeSysfs/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages": "1", + "/fakeSysfs/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages": "1", + } + fakeSys.SetHugePagesNr(hugePageNr, nil) + + nodes, cores, err := GetNodesInfo(fakeSys) + assert.NotNil(t, err) + assert.Equal(t, []info.Node([]info.Node(nil)), nodes) + assert.Equal(t, 0, cores) +} + +func TestGetNodesInfoWithoutHugePagesInfo(t *testing.T) { + fakeSys := &fakesysfs.FakeSysFs{} + c := sysfs.CacheInfo{ + Size: 32 * 1024, + Type: "unified", + Level: 2, + Cpus: 2, + } + fakeSys.SetCacheInfo(c) + + nodesPaths := []string{ + "/fakeSysfs/devices/system/node/node0", + "/fakeSysfs/devices/system/node/node1", + } + fakeSys.SetNodesPaths(nodesPaths, nil) + + cpusPaths := map[string][]string{ + "/fakeSysfs/devices/system/node/node0": { + "/fakeSysfs/devices/system/node/node0/cpu0", + "/fakeSysfs/devices/system/node/node0/cpu1", + }, + "/fakeSysfs/devices/system/node/node1": { + "/fakeSysfs/devices/system/node/node0/cpu2", + "/fakeSysfs/devices/system/node/node0/cpu3", + }, + } + fakeSys.SetCPUsPaths(cpusPaths, nil) + + coreThread := map[string]string{ + "/fakeSysfs/devices/system/node/node0/cpu0": "0", + "/fakeSysfs/devices/system/node/node0/cpu1": "0", + "/fakeSysfs/devices/system/node/node0/cpu2": "1", + "/fakeSysfs/devices/system/node/node0/cpu3": "1", + } + fakeSys.SetCoreThreads(coreThread, nil) + + memTotal := "MemTotal: 32817192 kB" + fakeSys.SetMemory(memTotal, nil) + + nodes, cores, err := GetNodesInfo(fakeSys) + assert.Nil(t, err) + assert.Equal(t, 2, len(nodes)) + assert.Equal(t, 4, cores) + + nodesJSON, err := json.Marshal(nodes) + assert.Nil(t, err) + expectedNodes := ` + [ + { + "node_id": 0, + "memory": 33604804608, + "hugepages": null, + "cores": [ + { + "core_id": 0, + "thread_ids": [ + 0, + 1 + ], + "caches": [ + { + "size": 32768, + "type": "unified", + "level": 2 + } + ] + } + ], + "caches": null + }, + { + "node_id": 1, + "memory": 33604804608, + "hugepages": null, + "cores": [ + { + "core_id": 1, + "thread_ids": [ + 2, + 3 + ], + "caches": [ + { + "size": 32768, + "type": "unified", + "level": 2 + } + ] + } + ], + "caches": null + } + ]` + assert.JSONEq(t, expectedNodes, string(nodesJSON)) +} + +func TestGetNodeMemInfo(t *testing.T) { + fakeSys := &fakesysfs.FakeSysFs{} + memTotal := "MemTotal: 32817192 kB" + fakeSys.SetMemory(memTotal, nil) + + mem, err := getNodeMemInfo(fakeSys, "/fakeSysfs/devices/system/node/node0") + assert.Nil(t, err) + assert.Equal(t, uint64(32817192*1024), mem) +} + +func TestGetNodeMemInfoWithMissingMemTotaInMemInfo(t *testing.T) { + fakeSys := &fakesysfs.FakeSysFs{} + memTotal := "MemXXX: 32817192 kB" + fakeSys.SetMemory(memTotal, nil) + + mem, err := getNodeMemInfo(fakeSys, "/fakeSysfs/devices/system/node/node0") + assert.NotNil(t, err) + assert.Equal(t, uint64(0), mem) +} + +func TestGetNodeMemInfoWhenMemInfoMissing(t *testing.T) { + fakeSys := &fakesysfs.FakeSysFs{} + memTotal := "" + fakeSys.SetMemory(memTotal, fmt.Errorf("Cannot read meminfo file")) + + mem, err := getNodeMemInfo(fakeSys, "/fakeSysfs/devices/system/node/node0") + assert.Nil(t, err) + assert.Equal(t, uint64(0), mem) +} + +func TestGetCoresInfoWhenCoreIDIsNotDigit(t *testing.T) { + sysFs := &fakesysfs.FakeSysFs{} + nodesPaths := []string{ + "/fakeSysfs/devices/system/node/node0", + } + sysFs.SetNodesPaths(nodesPaths, nil) + + cpusPaths := map[string][]string{ + "/fakeSysfs/devices/system/node/node0": { + "/fakeSysfs/devices/system/node/node0/cpu0", + }, + } + sysFs.SetCPUsPaths(cpusPaths, nil) + + coreThread := map[string]string{ + "/fakeSysfs/devices/system/node/node0/cpu0": "abc", + } + sysFs.SetCoreThreads(coreThread, nil) + + cores, coreID, err := getCoresInfo(sysFs, "/fakeSysfs/devices/system/node/node0") + assert.NotNil(t, err) + assert.Equal(t, []info.Core(nil), cores) + assert.Equal(t, 0, coreID) +} + func TestGetBlockDeviceInfo(t *testing.T) { fakeSys := fakesysfs.FakeSysFs{} disks, err := GetBlockDeviceInfo(&fakeSys)