cgroups v2: memory stats

cgroup/cgroup.go

@@ -61,95 +61,15 @@ func (t ContainerType) String() string {
 	}
 }
 
-type Stats struct {
-	CpuUsageSeconds      float64
-	ThrottledTimeSeconds float64
-	CpuQuotaCores        float64
-
-	MemoryRssBytes   float64
-	MemoryCacheBytes float64
-	MemoryLimitBytes float64
-
-	Blkio map[string]BlkioStat
-}
-
 type Cgroup struct {
 	Id            string
 	Version       Version
 	ContainerType ContainerType
 	ContainerId   string
 
-	prevStats *Stats
-
 	subsystems map[string]string
 }
 
-func (cg *Cgroup) GetStats() *Stats {
-	cur, err := cg.getCurrentStats()
-	if err != nil {
-		if !common.IsNotExist(err) {
-			klog.Warningf("failed to get cgroup stats: %s", err)
-		}
-		return nil
-	}
-	var res *Stats
-	if cg.prevStats != nil {
-		res = &Stats{
-			CpuUsageSeconds:      cur.CpuUsageSeconds - cg.prevStats.CpuUsageSeconds,
-			ThrottledTimeSeconds: cur.ThrottledTimeSeconds - cg.prevStats.ThrottledTimeSeconds,
-			CpuQuotaCores:        cur.CpuQuotaCores,
-			MemoryRssBytes:       cur.MemoryRssBytes,
-			MemoryCacheBytes:     cur.MemoryCacheBytes,
-			MemoryLimitBytes:     cur.MemoryLimitBytes,
-			Blkio:                map[string]BlkioStat{},
-		}
-		for majorMinor, stat := range cur.Blkio {
-			prev, ok := cg.prevStats.Blkio[majorMinor]
-			if !ok {
-				continue
-			}
-			res.Blkio[majorMinor] = BlkioStat{
-				ReadOps:      stat.ReadOps - prev.ReadOps,
-				WriteOps:     stat.WriteOps - prev.WriteOps,
-				ReadBytes:    stat.ReadBytes - prev.ReadBytes,
-				WrittenBytes: stat.WrittenBytes - prev.WrittenBytes,
-			}
-		}
-	}
-	cg.prevStats = cur
-	return res
-}
-
-func (cg *Cgroup) getCurrentStats() (*Stats, error) {
-	stats := &Stats{}
-	var err error
-	if stats.CpuUsageSeconds, err = cg.CpuUsageSeconds(); err != nil {
-		return nil, err
-	}
-	if stats.ThrottledTimeSeconds, err = cg.ThrottledTimeSeconds(); err != nil {
-		return nil, err
-	}
-	if stats.CpuQuotaCores, err = cg.CpuQuotaCores(); err != nil {
-		return nil, err
-	}
-	m, err := cg.MemoryStat()
-	if err != nil {
-		return nil, err
-	}
-	stats.MemoryRssBytes = float64(m.RSS)
-	stats.MemoryCacheBytes = float64(m.Cache)
-	l, err := cg.MemoryLimitBytes()
-	if err != nil {
-		return nil, err
-	}
-	stats.MemoryLimitBytes = float64(l)
-	stats.Blkio, err = cg.BlkioStat()
-	if err != nil {
-		return nil, err
-	}
-	return stats, nil
-}
-
 func (cg *Cgroup) CreatedAt() time.Time {
 	fi, err := os.Stat(path.Join(cgRoot, "cpu", cg.subsystems["cpu"]))
 	if err != nil {

cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/memory.current

@@ -0,0 +1 @@
+48648192

cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/memory.max

@@ -0,0 +1 @@
+max

cgroup/fixtures/cgroup/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod8712f785_1a3e_41ec_a00b_e2dcc77431cb.slice/docker-73051af271105c07e1f493b34856a77e665e3b0b4fc72f76c807dfbffeb881bd.scope/memory.stat

@@ -0,0 +1,40 @@
+anon 44892160
+file 1044480
+kernel_stack 262144
+pagetables 536576
+percpu 0
+sock 4096
+shmem 98304
+file_mapped 0
+file_dirty 4096
+file_writeback 0
+swapcached 0
+anon_thp 0
+file_thp 0
+shmem_thp 0
+inactive_anon 44888064
+active_anon 102400
+inactive_file 159744
+active_file 786432
+unevictable 0
+slab_reclaimable 1627544
+slab_unreclaimable 264968
+slab 1892512
+workingset_refault_anon 0
+workingset_refault_file 0
+workingset_activate_anon 0
+workingset_activate_file 0
+workingset_restore_anon 0
+workingset_restore_file 0
+workingset_nodereclaim 0
+pgfault 29601375
+pgmajfault 0
+pgrefill 0
+pgscan 0
+pgsteal 0
+pgactivate 6143
+pgdeactivate 0
+pglazyfree 0
+pglazyfreed 0
+thp_fault_alloc 0
+thp_collapse_alloc 0

cgroup/fixtures/cgroup/memory/system.slice/ssh.service/memory.stat

@@ -0,0 +1,32 @@
+cache 3206844416
+rss 12778020864
+rss_huge 11370758144
+mapped_file 1997103104
+dirty 31039488
+writeback 0
+pgpgin 5542721218
+pgpgout 5566438724
+pgfault 4304414697
+pgmajfault 44816508
+inactive_anon 0
+active_anon 12777906176
+inactive_file 1601998848
+active_file 1604784128
+unevictable 0
+hierarchical_memory_limit 21474836480
+total_cache 3206844416
+total_rss 12778020864
+total_rss_huge 11370758144
+total_mapped_file 1997103104
+total_dirty 31039488
+total_writeback 0
+total_pgpgin 5542721218
+total_pgpgout 5566438724
+total_pgfault 4304414697
+total_pgmajfault 44816508
+total_inactive_anon 0
+total_active_anon 12777906176
+total_inactive_file 1601998848
+total_active_file 1604784128
+total_unevictable 0
+

cgroup/fixtures/cgroup/system.slice/docker-ba7b10d15d16e10e3de7a2dcd408a3d971169ae303f46cfad4c5453c6326fee2.scope/memory.current

@@ -0,0 +1 @@
+131047424

cgroup/fixtures/cgroup/system.slice/docker-ba7b10d15d16e10e3de7a2dcd408a3d971169ae303f46cfad4c5453c6326fee2.scope/memory.max

@@ -0,0 +1 @@
+4294967296

cgroup/fixtures/cgroup/system.slice/docker-ba7b10d15d16e10e3de7a2dcd408a3d971169ae303f46cfad4c5453c6326fee2.scope/memory.stat

@@ -0,0 +1,40 @@
+anon 75247616
+file 50835456
+kernel_stack 180224
+pagetables 573440
+percpu 0
+sock 0
+shmem 0
+file_mapped 4038656
+file_dirty 49152
+file_writeback 0
+swapcached 0
+anon_thp 2097152
+file_thp 0
+shmem_thp 0
+inactive_anon 77389824
+active_anon 4096
+inactive_file 50204672
+active_file 630784
+unevictable 0
+slab_reclaimable 1889688
+slab_unreclaimable 154200
+slab 2043888
+workingset_refault_anon 0
+workingset_refault_file 0
+workingset_activate_anon 0
+workingset_activate_file 0
+workingset_restore_anon 0
+workingset_restore_file 0
+workingset_nodereclaim 0
+pgfault 9245661
+pgmajfault 1
+pgrefill 0
+pgscan 0
+pgsteal 0
+pgactivate 2173
+pgdeactivate 0
+pglazyfree 0
+pglazyfreed 0
+thp_fault_alloc 208
+thp_collapse_alloc 55

cgroup/memory.go

@@ -9,12 +9,27 @@ const maxMemory = 1 << 62
 type MemoryStat struct {
 	RSS   uint64
 	Cache uint64
+	Limit uint64
 }
 
-func (cg *Cgroup) MemoryStat() (MemoryStat, error) {
+func (cg *Cgroup) MemoryStat() (*MemoryStat, error) {
+	if cg.Version == V1 {
+		return cg.memoryStatV1()
+	}
+	return cg.memoryStatV2()
+}
+
+func (cg *Cgroup) memoryStatV1() (*MemoryStat, error) {
 	vars, err := readVariablesFromFile(path.Join(cgRoot, "memory", cg.subsystems["memory"], "memory.stat"))
 	if err != nil {
-		return MemoryStat{}, err
+		return nil, err
+	}
+	limit, err := readUintFromFile(path.Join(cgRoot, "memory", cg.subsystems["memory"], "memory.limit_in_bytes"))
+	if err != nil {
+		return nil, err
+	}
+	if limit > maxMemory {
+		limit = 0
 	}
 	// Note from https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt:
 	// Only anonymous and swap cache memory is listed as part of 'rss' stat.
@@ -24,19 +39,26 @@ func (cg *Cgroup) MemoryStat() (MemoryStat, error) {
 	//	(Note: file and shmem may be shared among other cgroups. In that case,
 	//	 mapped_file is accounted only when the memory cgroup is owner of page
 	//	 cache.)
-	return MemoryStat{
+	return &MemoryStat{
 		RSS:   vars["rss"] + vars["mapped_file"],
 		Cache: vars["cache"],
+		Limit: limit,
 	}, nil
 }
 
-func (cg *Cgroup) MemoryLimitBytes() (uint64, error) {
-	limit, err := readUintFromFile(path.Join(cgRoot, "memory", cg.subsystems["memory"], "memory.limit_in_bytes"))
+func (cg *Cgroup) memoryStatV2() (*MemoryStat, error) {
+	current, err := readUintFromFile(path.Join(cgRoot, cg.subsystems[""], "memory.current"))
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	if limit > maxMemory {
-		return 0, nil
+	vars, err := readVariablesFromFile(path.Join(cgRoot, cg.subsystems[""], "memory.stat"))
+	if err != nil {
+		return nil, err
 	}
-	return limit, nil
+	limit, _ := readUintFromFile(path.Join(cgRoot, cg.subsystems[""], "memory.max"))
+	return &MemoryStat{
+		RSS:   current - vars["file"],
+		Cache: vars["file"],
+		Limit: limit,
+	}, nil
 }

cgroup/memory_test.go

@@ -9,24 +9,30 @@ import (
 func TestCgroup_MemoryStat(t *testing.T) {
 	cgRoot = "fixtures/cgroup"
 
-	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/200/cgroup"))
+	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/100/cgroup"))
 	stat, err := cg.MemoryStat()
 	assert.Nil(t, err)
+	assert.Equal(t, uint64(0), stat.Limit)
+
+	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/200/cgroup"))
+	stat, err = cg.MemoryStat()
+	assert.Nil(t, err)
 	assert.Equal(t, uint64(14775123968), stat.RSS)
 	assert.Equal(t, uint64(3206844416), stat.Cache)
-}
+	assert.Equal(t, uint64(21474836480), stat.Limit)
 
-func TestCgroup_MemoryLimitBytes(t *testing.T) {
-	cgRoot = "fixtures/cgroup"
-
-	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/100/cgroup"))
-	limit, err := cg.MemoryLimitBytes()
+	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/400/cgroup"))
+	stat, err = cg.MemoryStat()
 	assert.Nil(t, err)
-	assert.Equal(t, uint64(0), limit)
+	assert.Equal(t, uint64(48648192-1044480), stat.RSS)
+	assert.Equal(t, uint64(1044480), stat.Cache)
+	assert.Equal(t, uint64(0), stat.Limit)
 
-	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/200/cgroup"))
-	limit, err = cg.MemoryLimitBytes()
+	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/500/cgroup"))
+	stat, err = cg.MemoryStat()
 	assert.Nil(t, err)
-	assert.Equal(t, uint64(21474836480), limit)
+	assert.Equal(t, uint64(131047424-50835456), stat.RSS)
+	assert.Equal(t, uint64(50835456), stat.Cache)
+	assert.Equal(t, uint64(4294967296), stat.Limit)
 
 }

containers/container.go

@@ -176,12 +176,12 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 		ch <- counter(metrics.DiskDelay, float64(c.delays.disk)/float64(time.Second))
 	}
 
-	if v, err := c.cgroup.MemoryLimitBytes(); err == nil && v > 0 {
-		ch <- gauge(metrics.MemoryLimit, float64(v))
-	}
 	if s, err := c.cgroup.MemoryStat(); err == nil {
 		ch <- gauge(metrics.MemoryRss, float64(s.RSS))
 		ch <- gauge(metrics.MemoryCache, float64(s.Cache))
+		if s.Limit > 0 {
+			ch <- gauge(metrics.MemoryLimit, float64(s.Limit))
+		}
 	}
 
 	if c.oomKills > 0 {