cgroups v2: CPU stats

cgroup/cpu.go

@@ -1,42 +1,84 @@
 package cgroup
 
 import (
+	"fmt"
+	"io/ioutil"
 	"path"
+	"strconv"
+	"strings"
 )
 
-type ThrottlingStat struct {
-	Periods              uint64
-	ThrottledPeriods     uint64
+type CPUStat struct {
+	UsageSeconds         float64
 	ThrottledTimeSeconds float64
+	LimitCores           float64
 }
 
-func (cg Cgroup) ThrottledTimeSeconds() (float64, error) {
-	vars, err := readVariablesFromFile(path.Join(cgRoot, "cpu", cg.subsystems["cpu"], "cpu.stat"))
-	if err != nil {
-		return 0, err
+func (cg Cgroup) CpuStat() (*CPUStat, error) {
+	if cg.Version == V1 {
+		return cg.cpuStatV1()
 	}
-	return float64(vars["throttled_time"]) / 1e9, nil
+	return cg.cpuStatV2()
 }
 
-func (cg Cgroup) CpuUsageSeconds() (float64, error) {
+func (cg Cgroup) cpuStatV1() (*CPUStat, error) {
+	throttling, err := readVariablesFromFile(path.Join(cgRoot, "cpu", cg.subsystems["cpu"], "cpu.stat"))
+	if err != nil {
+		return nil, err
+	}
 	usageNs, err := readIntFromFile(path.Join(cgRoot, "cpuacct", cg.subsystems["cpuacct"], "cpuacct.usage"))
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	return float64(usageNs) / 1e9, err
-}
-
-func (cg Cgroup) CpuQuotaCores() (float64, error) {
 	periodUs, err := readIntFromFile(path.Join(cgRoot, "cpu", cg.subsystems["cpu"], "cpu.cfs_period_us"))
 	if err != nil {
-		return -1, err
+		return nil, err
 	}
 	quotaUs, err := readIntFromFile(path.Join(cgRoot, "cpu", cg.subsystems["cpu"], "cpu.cfs_quota_us"))
 	if err != nil {
-		return -1, err
+		return nil, err
+	}
+	res := &CPUStat{
+		UsageSeconds:         float64(usageNs) / 1e9,
+		ThrottledTimeSeconds: float64(throttling["throttled_time"]) / 1e9,
+	}
+	if quotaUs > 0 {
+		res.LimitCores = float64(quotaUs) / float64(periodUs)
+	}
+	return res, nil
+}
+
+func (cg Cgroup) cpuStatV2() (*CPUStat, error) {
+	vars, err := readVariablesFromFile(path.Join(cgRoot, cg.subsystems[""], "cpu.stat"))
+	if err != nil {
+		return nil, err
+	}
+	res := &CPUStat{
+		UsageSeconds:         float64(vars["usage_usec"]) / 1e6,
+		ThrottledTimeSeconds: float64(vars["throttled_usec"]) / 1e6,
+	}
+	payload, err := ioutil.ReadFile(path.Join(cgRoot, cg.subsystems[""], "cpu.max"))
+	if err != nil {
+		return nil, err
+	}
+	data := strings.TrimSpace(string(payload))
+	parts := strings.Fields(data)
+	if len(parts) != 2 {
+		return nil, fmt.Errorf("invalid cpu.max payload: %s", data)
+	}
+	if parts[0] == "max" { //no limit
+		return res, nil
+	}
+	quotaUs, err := strconv.ParseUint(parts[0], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("invalid quota value in cpu.max: %s", parts[0])
+	}
+	periodUs, err := strconv.ParseUint(parts[1], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("invalid period value in cpu.max: %s", parts[1])
 	}
-	if quotaUs < 0 {
-		return -1, nil
+	if periodUs > 0 {
+		res.LimitCores = float64(quotaUs) / float64(periodUs)
 	}
-	return float64(quotaUs) / float64(periodUs), nil
+	return res, nil
 }

cgroup/cpu_test.go

@@ -6,34 +6,32 @@ import (
 	"testing"
 )
 
-func TestCgroup_CpuQuotaCores(t *testing.T) {
+func TestCgroup_CpuStat(t *testing.T) {
 	cgRoot = "fixtures/cgroup"
 
 	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/100/cgroup"))
-	quota, err := cg.CpuQuotaCores()
+	s, err := cg.CpuStat()
 	assert.Nil(t, err)
-	assert.Equal(t, -1., quota)
+	assert.Equal(t, 0., s.LimitCores)
+	assert.Equal(t, 26778.913419246, s.UsageSeconds)
 
 	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/200/cgroup"))
-	quota, err = cg.CpuQuotaCores()
+	s, err = cg.CpuStat()
 	assert.Nil(t, err)
-	assert.Equal(t, 1.5, quota)
-}
-
-func TestCgroup_CpuUsageSeconds(t *testing.T) {
-	cgRoot = "fixtures/cgroup"
+	assert.Equal(t, 1.5, s.LimitCores)
+	assert.Equal(t, 254005.032764376, s.ThrottledTimeSeconds)
 
-	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/100/cgroup"))
-	usage, err := cg.CpuUsageSeconds()
+	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/400/cgroup"))
+	s, err = cg.CpuStat()
 	assert.Nil(t, err)
-	assert.Equal(t, 26778.913419246, usage)
-}
-
-func TestCgroup_ThrottlingStat(t *testing.T) {
-	cgRoot = "fixtures/cgroup"
+	assert.Equal(t, 0.1, s.LimitCores)
+	assert.Equal(t, 0.363166, s.ThrottledTimeSeconds)
+	assert.Equal(t, 3795.681254, s.UsageSeconds)
 
-	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/200/cgroup"))
-	tt, err := cg.ThrottledTimeSeconds()
+	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/500/cgroup"))
+	s, err = cg.CpuStat()
 	assert.Nil(t, err)
-	assert.Equal(t, 254005.032764376, tt)
+	assert.Equal(t, 0., s.LimitCores)
+	assert.Equal(t, 0., s.ThrottledTimeSeconds)
+	assert.Equal(t, 5531.521992, s.UsageSeconds)
 }

cgroup/fixtures/cgroup/cpuacct/docker/b43d92bf1e5c6f78bb9b7bc6f40721280299855ba692092716e3a1b6c0b86f3f/cpuacct.usage

@@ -0,0 +1 @@
+107292866812141

cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/cpu.max

@@ -0,0 +1 @@
+10000 100000

cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/cpu.stat

@@ -0,0 +1,6 @@
+usage_usec 3795681254
+user_usec 3160014031
+system_usec 635667223
+nr_periods 809036
+nr_throttled 76
+throttled_usec 363166

cgroup/fixtures/cgroup/system.slice/docker-ba7b10d15d16e10e3de7a2dcd408a3d971169ae303f46cfad4c5453c6326fee2.scope/cpu.max

@@ -0,0 +1 @@
+max 100000

cgroup/fixtures/cgroup/system.slice/docker-ba7b10d15d16e10e3de7a2dcd408a3d971169ae303f46cfad4c5453c6326fee2.scope/cpu.stat

@@ -0,0 +1,6 @@
+usage_usec 5531521992
+user_usec 3642846968
+system_usec 1888675024
+nr_periods 0
+nr_throttled 0
+throttled_usec 0

containers/container.go

@@ -160,14 +160,12 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 
 	ch <- counter(metrics.Restarts, float64(c.restarts))
 
-	if v, err := c.cgroup.CpuQuotaCores(); err == nil && v > 0 {
-		ch <- gauge(metrics.CPULimit, v)
-	}
-	if v, err := c.cgroup.CpuUsageSeconds(); err == nil {
-		ch <- counter(metrics.CPUUsage, v)
-	}
-	if v, err := c.cgroup.ThrottledTimeSeconds(); err == nil {
-		ch <- counter(metrics.ThrottledTime, v)
+	if cpu, err := c.cgroup.CpuStat(); err == nil {
+		if cpu.LimitCores > 0 {
+			ch <- gauge(metrics.CPULimit, cpu.LimitCores)
+		}
+		ch <- counter(metrics.CPUUsage, cpu.UsageSeconds)
+		ch <- counter(metrics.ThrottledTime, cpu.ThrottledTimeSeconds)
 	}
 
 	if taskstatsClient != nil {