Merge pull request #2414 from katarzyna-z/kk-machine-api-cores-sockets

Extending machine API by number of physical cpu cores and cpu sockets

info/v1/machine.go

@@ -161,6 +161,12 @@ type MachineInfo struct {
 	// The number of cores in this machine.
 	NumCores int `json:"num_cores"`
 
+	// The number of physical cores in this machine.
+	NumPhysicalCores int `json:"num_physical_cores"`
+
+	// The number of cpu sockets in this machine.
+	NumSockets int `json:"num_sockets"`
+
 	// Maximum clock speed for the cores, in KHz.
 	CpuFrequency uint64 `json:"cpu_frequency_khz"`
 

machine/info.go

@@ -108,19 +108,21 @@ func Info(sysFs sysfs.SysFs, fsInfo fs.FsInfo, inHostNamespace bool) (*info.Mach
 	instanceID := realCloudInfo.GetInstanceID()
 
 	machineInfo := &info.MachineInfo{
-		NumCores:       numCores,
+		NumCores:         numCores,
-		CpuFrequency:   clockSpeed,
+		NumPhysicalCores: GetPhysicalCores(cpuinfo),
-		MemoryCapacity: memoryCapacity,
+		NumSockets:       GetSockets(cpuinfo),
-		HugePages:      hugePagesInfo,
+		CpuFrequency:     clockSpeed,
-		DiskMap:        diskMap,
+		MemoryCapacity:   memoryCapacity,
-		NetworkDevices: netDevices,
+		HugePages:        hugePagesInfo,
-		Topology:       topology,
+		DiskMap:          diskMap,
-		MachineID:      getInfoFromFiles(filepath.Join(rootFs, *machineIdFilePath)),
+		NetworkDevices:   netDevices,
-		SystemUUID:     systemUUID,
+		Topology:         topology,
-		BootID:         getInfoFromFiles(filepath.Join(rootFs, *bootIdFilePath)),
+		MachineID:        getInfoFromFiles(filepath.Join(rootFs, *machineIdFilePath)),
-		CloudProvider:  cloudProvider,
+		SystemUUID:       systemUUID,
-		InstanceType:   instanceType,
+		BootID:           getInfoFromFiles(filepath.Join(rootFs, *bootIdFilePath)),
-		InstanceID:     instanceID,
+		CloudProvider:    cloudProvider,
+		InstanceType:     instanceType,
+		InstanceID:       instanceID,
 	}
 
 	for i := range filesystems {

machine/machine.go

@@ -39,18 +39,50 @@ import (
 
 var (
 	cpuRegExp     = regexp.MustCompile(`^processor\s*:\s*([0-9]+)$`)
-	coreRegExp    = regexp.MustCompile(`^core id\s*:\s*([0-9]+)$`)
+	coreRegExp    = regexp.MustCompile(`(?m)^core id\s*:\s*([0-9]+)$`)
-	nodeRegExp    = regexp.MustCompile(`^physical id\s*:\s*([0-9]+)$`)
+	nodeRegExp    = regexp.MustCompile(`(?m)^physical id\s*:\s*([0-9]+)$`)
 	nodeBusRegExp = regexp.MustCompile(`^node([0-9]+)$`)
 	// Power systems have a different format so cater for both
 	cpuClockSpeedMHz     = regexp.MustCompile(`(?:cpu MHz|clock)\s*:\s*([0-9]+\.[0-9]+)(?:MHz)?`)
 	memoryCapacityRegexp = regexp.MustCompile(`MemTotal:\s*([0-9]+) kB`)
 	swapCapacityRegexp   = regexp.MustCompile(`SwapTotal:\s*([0-9]+) kB`)
+
+	cpuBusPath = "/sys/bus/cpu/devices/"
 )
 
 const maxFreqFile = "/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq"
-const cpuBusPath = "/sys/bus/cpu/devices/"
 const nodePath = "/sys/devices/system/node"
+const sysFsCPUCoreID = "core_id"
+const sysFsCPUPhysicalPackageID = "physical_package_id"
+const sysFsCPUTopology = "topology"
+
+// GetPhysicalCores returns number of CPU cores reading /proc/cpuinfo file or if needed information from sysfs cpu path
+func GetPhysicalCores(procInfo []byte) int {
+	numCores := getUniqueMatchesCount(string(procInfo), coreRegExp)
+	if numCores == 0 {
+		// read number of cores from /sys/bus/cpu/devices/cpu*/topology/core_id to deal with processors
+		// for which 'core id' is not available in /proc/cpuinfo
+		numCores = getUniqueCPUPropertyCount(cpuBusPath, sysFsCPUCoreID)
+	}
+	if numCores == 0 {
+		klog.Errorf("Cannot read number of physical cores correctly, number of cores set to %d", numCores)
+	}
+	return numCores
+}
+
+// GetSockets returns number of CPU sockets reading /proc/cpuinfo file or if needed information from sysfs cpu path
+func GetSockets(procInfo []byte) int {
+	numSocket := getUniqueMatchesCount(string(procInfo), nodeRegExp)
+	if numSocket == 0 {
+		// read number of sockets from /sys/bus/cpu/devices/cpu*/topology/physical_package_id to deal with processors
+		// for which 'physical id' is not available in /proc/cpuinfo
+		numSocket = getUniqueCPUPropertyCount(cpuBusPath, sysFsCPUPhysicalPackageID)
+	}
+	if numSocket == 0 {
+		klog.Errorf("Cannot read number of sockets correctly, number of sockets set to %d", numSocket)
+	}
+	return numSocket
+}
 
 // GetClockSpeed returns the CPU clock speed, given a []byte formatted as the /proc/cpuinfo file.
 func GetClockSpeed(procInfo []byte) (uint64, error) {
@@ -132,11 +164,11 @@ func parseCapacity(b []byte, r *regexp.Regexp) (uint64, error) {
 	return m * 1024, err
 }
 
-/* Look for sysfs cpu path containing core_id */
+// Looks for sysfs cpu path containing core_id
-/* Such as: sys/bus/cpu/devices/cpu0/topology/core_id */
+// Such as: sys/bus/cpu/devices/cpu0/topology/core_id
 func getCoreIdFromCpuBus(cpuBusPath string, threadId int) (int, error) {
 	path := filepath.Join(cpuBusPath, fmt.Sprintf("cpu%d/topology", threadId))
-	file := filepath.Join(path, "core_id")
+	file := filepath.Join(path, sysFsCPUCoreID)
 
 	num, err := ioutil.ReadFile(file)
 	if err != nil {
@@ -156,8 +188,30 @@ func getCoreIdFromCpuBus(cpuBusPath string, threadId int) (int, error) {
 	return int(coreId), nil
 }
 
-/* Look for sysfs cpu path containing node id */
+// Looks for sysfs cpu path containing given CPU property, e.g. core_id or physical_package_id
-/* Such as: /sys/bus/cpu/devices/cpu0/node%d */
+// and returns number of unique values of given property, exemplary usage: getting number of CPU physical cores
+func getUniqueCPUPropertyCount(cpuBusPath string, propertyName string) int {
+	pathPattern := cpuBusPath + "cpu*[0-9]"
+	sysCPUPaths, err := filepath.Glob(pathPattern)
+	if err != nil {
+		klog.Errorf("Cannot find files matching pattern (pathPattern: %s),  number of unique %s set to 0", pathPattern, propertyName)
+		return 0
+	}
+	uniques := make(map[string]bool)
+	for _, sysCPUPath := range sysCPUPaths {
+		propertyPath := filepath.Join(sysCPUPath, sysFsCPUTopology, propertyName)
+		propertyVal, err := ioutil.ReadFile(propertyPath)
+		if err != nil {
+			klog.Errorf("Cannot open %s, number of unique %s  set to 0", propertyPath, propertyName)
+			return 0
+		}
+		uniques[string(propertyVal)] = true
+	}
+	return len(uniques)
+}
+
+// Looks for sysfs cpu path containing node id
+// Such as: /sys/bus/cpu/devices/cpu0/node%d
 func getNodeIdFromCpuBus(cpuBusPath string, threadId int) (int, error) {
 	path := filepath.Join(cpuBusPath, fmt.Sprintf("cpu%d", threadId))
 
@@ -354,6 +408,16 @@ func extractValue(s string, r *regexp.Regexp) (bool, int, error) {
 	return false, -1, nil
 }
 
+// getUniqueMatchesCount returns number of unique matches in given argument using provided regular expression
+func getUniqueMatchesCount(s string, r *regexp.Regexp) int {
+	matches := r.FindAllString(s, -1)
+	uniques := make(map[string]bool)
+	for _, match := range matches {
+		uniques[match] = true
+	}
+	return len(uniques)
+}
+
 func findNode(nodes []info.Node, id int) (bool, int) {
 	for i, n := range nodes {
 		if n.Id == id {

machine/testdata/cpu0/topology/physical_package_id

@@ -0,0 +1 @@
+0

machine/testdata/cpu9999/topology/physical_package_id

@@ -0,0 +1 @@
+0

machine/testdata/cpuinfo_arm

@@ -0,0 +1,43 @@
+processor    : 0
+model name    : ARMv7 Processor rev 4 (v7l)
+BogoMIPS    : 76.80
+Features    : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae evtstrm crc32
+CPU implementer    : 0x41
+CPU architecture: 7
+CPU variant    : 0x0
+CPU part    : 0xd03
+CPU revision    : 4
+
+processor    : 1
+model name    : ARMv7 Processor rev 4 (v7l)
+BogoMIPS    : 76.80
+Features    : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae evtstrm crc32
+CPU implementer    : 0x41
+CPU architecture: 7
+CPU variant    : 0x0
+CPU part    : 0xd03
+CPU revision    : 4
+
+processor    : 2
+model name    : ARMv7 Processor rev 4 (v7l)
+BogoMIPS    : 76.80
+Features    : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae evtstrm crc32
+CPU implementer    : 0x41
+CPU architecture: 7
+CPU variant    : 0x0
+CPU part    : 0xd03
+CPU revision    : 4
+
+processor    : 3
+model name    : ARMv7 Processor rev 4 (v7l)
+BogoMIPS    : 76.80
+Features    : half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt vfpd32 lpae evtstrm crc32
+CPU implementer    : 0x41
+CPU architecture: 7
+CPU variant    : 0x0
+CPU part    : 0xd03
+CPU revision    : 4
+
+Hardware    : BCM2835
+Revision    : 0000
+Serial        : 00000000d0a71bfd

machine/topology_test.go

@@ -23,8 +23,79 @@ import (
 	info "github.com/google/cadvisor/info/v1"
 	"github.com/google/cadvisor/utils/sysfs"
 	"github.com/google/cadvisor/utils/sysfs/fakesysfs"
+	"github.com/stretchr/testify/assert"
 )
 
+func TestPhysicalCores(t *testing.T) {
+	testfile := "./testdata/cpuinfo"
+
+	testcpuinfo, err := ioutil.ReadFile(testfile)
+	assert.Nil(t, err)
+	assert.NotNil(t, testcpuinfo)
+
+	numPhysicalCores := GetPhysicalCores(testcpuinfo)
+	assert.Equal(t, 6, numPhysicalCores)
+}
+
+func TestPhysicalCoresReadingFromCpuBus(t *testing.T) {
+	cpuBusPath = "./testdata/"           // overwriting global variable to mock sysfs
+	testfile := "./testdata/cpuinfo_arm" // mock cpuinfo without core id
+
+	testcpuinfo, err := ioutil.ReadFile(testfile)
+	assert.Nil(t, err)
+	assert.NotNil(t, testcpuinfo)
+
+	numPhysicalCores := GetPhysicalCores(testcpuinfo)
+	assert.Equal(t, 2, numPhysicalCores)
+}
+
+func TestPhysicalCoresFromWrongSysFs(t *testing.T) {
+	cpuBusPath = "./testdata/wrongsysfs" // overwriting global variable to mock sysfs
+	testfile := "./testdata/cpuinfo_arm" // mock cpuinfo without core id
+
+	testcpuinfo, err := ioutil.ReadFile(testfile)
+	assert.Nil(t, err)
+	assert.NotNil(t, testcpuinfo)
+
+	numPhysicalCores := GetPhysicalCores(testcpuinfo)
+	assert.Equal(t, 0, numPhysicalCores)
+}
+
+func TestSockets(t *testing.T) {
+	testfile := "./testdata/cpuinfo"
+
+	testcpuinfo, err := ioutil.ReadFile(testfile)
+	assert.Nil(t, err)
+	assert.NotNil(t, testcpuinfo)
+
+	numSockets := GetSockets(testcpuinfo)
+	assert.Equal(t, 2, numSockets)
+}
+
+func TestSocketsReadingFromCpuBus(t *testing.T) {
+	cpuBusPath = "./testdata/wrongsysfs" // overwriting global variable to mock sysfs
+	testfile := "./testdata/cpuinfo_arm" // mock cpuinfo without physical id
+
+	testcpuinfo, err := ioutil.ReadFile(testfile)
+	assert.Nil(t, err)
+	assert.NotNil(t, testcpuinfo)
+
+	numSockets := GetSockets(testcpuinfo)
+	assert.Equal(t, 0, numSockets)
+}
+
+func TestSocketsReadingFromWrongSysFs(t *testing.T) {
+	cpuBusPath = "./testdata/"           // overwriting global variable to mock sysfs
+	testfile := "./testdata/cpuinfo_arm" // mock cpuinfo without physical id
+
+	testcpuinfo, err := ioutil.ReadFile(testfile)
+	assert.Nil(t, err)
+	assert.NotNil(t, testcpuinfo)
+
+	numSockets := GetSockets(testcpuinfo)
+	assert.Equal(t, 1, numSockets)
+}
+
 func TestTopology(t *testing.T) {
 	if runtime.GOARCH != "amd64" {
 		t.Skip("cpuinfo testdata is for amd64")