[lxc-devel] [lxd/master] Cgroup handling improvements

stgraber on Github lxc-bot at linuxcontainers.org
Sun Nov 8 04:41:18 UTC 2020


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 399 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20201107/0a56e78e/attachment.bin>
-------------- next part --------------
From 8f6e40a18fb2715f1888d9049058ad95afc2b192 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 6 Nov 2020 18:04:54 -0500
Subject: [PATCH 1/8] lxd/cgroup: Add file read/writer
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/cgroup/file.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 lxd/cgroup/file.go

diff --git a/lxd/cgroup/file.go b/lxd/cgroup/file.go
new file mode 100644
index 0000000000..7ad0679624
--- /dev/null
+++ b/lxd/cgroup/file.go
@@ -0,0 +1,90 @@
+package cgroup
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"github.com/lxc/lxd/shared"
+)
+
+// NewFileReadWriter returns a CGroup instance using the filesystem as its backend.
+func NewFileReadWriter(pid int, unifiedCapable bool) (*CGroup, error) {
+	// Setup the read/writer struct.
+	rw := fileReadWriter{}
+
+	// Locate the base path for each controller.
+	rw.paths = map[string]string{}
+
+	controllers, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid))
+	if err != nil {
+		return nil, err
+	}
+
+	for _, line := range strings.Split(string(controllers), "\n") {
+		// Skip empty lines.
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+
+		// Extract the fields.
+		fields := strings.Split(line, ":")
+
+		// Determine the mount path.
+		path := filepath.Join("/sys/fs/cgroup", fields[1], fields[2])
+		if fields[0] == "0" {
+			fields[1] = "unified"
+			if shared.PathExists("/sys/fs/cgroup/unified") {
+				path = filepath.Join("/sys/fs/cgroup", "unified", fields[2])
+			} else {
+				path = filepath.Join("/sys/fs/cgroup", fields[2])
+			}
+
+			if fields[2] != "/init.scope" {
+				path = filepath.Dir(path)
+			}
+		}
+
+		// Add the controllers individually.
+		for _, ctrl := range strings.Split(fields[1], ",") {
+			rw.paths[ctrl] = path
+		}
+	}
+
+	cg, err := New(&rw)
+	if err != nil {
+		return nil, err
+	}
+
+	cg.UnifiedCapable = unifiedCapable
+	return cg, nil
+}
+
+type fileReadWriter struct {
+	paths map[string]string
+}
+
+func (rw *fileReadWriter) Get(version Backend, controller string, key string) (string, error) {
+	path := filepath.Join(rw.paths[controller], key)
+	if cgLayout == CgroupsUnified {
+		path = filepath.Join(rw.paths["unified"], key)
+	}
+
+	value, err := ioutil.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSpace(string(value)), nil
+}
+
+func (rw *fileReadWriter) Set(version Backend, controller string, key string, value string) error {
+	path := filepath.Join(rw.paths[controller], key)
+	if cgLayout == CgroupsUnified {
+		path = filepath.Join(rw.paths["unified"], key)
+	}
+
+	return ioutil.WriteFile(path, []byte(value), 0600)
+}

From 71c25894921733922e305f6fdedfd18dbb26c3e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 6 Nov 2020 23:22:38 -0500
Subject: [PATCH 2/8] lxd/cgroup: Fix controller detection
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/cgroup/init.go | 69 +++++++++++++++++++++++++++-------------------
 1 file changed, 40 insertions(+), 29 deletions(-)

diff --git a/lxd/cgroup/init.go b/lxd/cgroup/init.go
index 732d4afa40..7338e6d47e 100644
--- a/lxd/cgroup/init.go
+++ b/lxd/cgroup/init.go
@@ -123,39 +123,46 @@ func (info *Info) SupportsVersion(resource Resource) (Backend, bool) {
 	switch resource {
 	case Blkio:
 		val, ok := cgControllers["blkio"]
-		if ok && val == V1 {
-			return V1, ok
+		if ok {
+			return val, ok
+		}
+
+		val, ok = cgControllers["io"]
+		if ok {
+			return val, ok
 		}
 
 		return Unavailable, false
 	case BlkioWeight:
 		val, ok := cgControllers["blkio.weight"]
-		if ok && val == V1 {
-			return V1, ok
+		if ok {
+			return val, ok
 		}
 
-		return Unavailable, false
-	case CPU:
-		val, ok := cgControllers["cpu"]
-		if ok && val == V1 {
-			return V1, ok
+		val, ok = cgControllers["io"]
+		if ok {
+			return val, ok
 		}
 
 		return Unavailable, false
+	case CPU:
+		val, ok := cgControllers["cpu"]
+		return val, ok
 	case CPUAcct:
 		val, ok := cgControllers["cpuacct"]
-		if ok && val == V1 {
-			return V1, ok
+		if ok {
+			return val, ok
 		}
 
-		return Unavailable, false
-	case CPUSet:
-		val, ok := cgControllers["cpuset"]
-		if ok && val == V1 {
-			return V1, ok
+		val, ok = cgControllers["cpu"]
+		if ok {
+			return val, ok
 		}
 
 		return Unavailable, false
+	case CPUSet:
+		val, ok := cgControllers["memory"]
+		return val, ok
 	case Devices:
 		val, ok := cgControllers["devices"]
 		return val, ok
@@ -185,8 +192,8 @@ func (info *Info) SupportsVersion(resource Resource) (Backend, bool) {
 		return Unavailable, false
 	case MemorySwapMaxUsage:
 		val, ok := cgControllers["memory.memsw.max_usage_in_bytes"]
-		if ok && val == V1 {
-			return V1, ok
+		if ok {
+			return val, ok
 		}
 
 		return Unavailable, false
@@ -204,22 +211,18 @@ func (info *Info) SupportsVersion(resource Resource) (Backend, bool) {
 		return Unavailable, false
 	case MemorySwappiness:
 		val, ok := cgControllers["memory.swappiness"]
-		if ok && val == V1 {
-			return V1, ok
+		if ok {
+			return val, ok
 		}
 
 		return Unavailable, false
 	case NetPrio:
 		val, ok := cgControllers["net_prio"]
-		if ok && val == V1 {
-			return V1, ok
-		}
-
-		return Unavailable, false
+		return val, ok
 	case Pids:
 		val, ok := cgControllers["pids"]
-		if ok && val == V1 {
-			return V1, ok
+		if ok {
+			return val, ok
 		}
 
 		return Unavailable, false
@@ -413,11 +416,11 @@ func init() {
 
 	val, ok = cgControllers["memory"]
 	if ok && val == V2 {
-		if shared.PathExists("/sys/fs/cgroup/memory.swap.max") {
+		if shared.PathExists("/sys/fs/cgroup/init.scope/memory.swap.max") {
 			cgControllers["memory.swap.max"] = V2
 		}
 
-		if shared.PathExists("/sys/fs/cgroup/memory.swap.current") {
+		if shared.PathExists("/sys/fs/cgroup/init.scope/memory.swap.current") {
 			cgControllers["memory.swap.current"] = V2
 		}
 	}
@@ -429,4 +432,12 @@ func init() {
 	} else if hasV2 {
 		cgLayout = CgroupsUnified
 	}
+
+	if cgLayout == CgroupsUnified {
+		// With Cgroup2 devices is built-in (through eBPF).
+		cgControllers["devices"] = V2
+
+		// With Cgroup2 freezer is built-in.
+		cgControllers["freezer"] = V2
+	}
 }

From fa5f713cacad1e79b151b2d4f27a6858a92c5c56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 6 Nov 2020 18:04:44 -0500
Subject: [PATCH 3/8] lxd/cgroup: Add cpuset functions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/cgroup/abstraction.go | 45 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 44 insertions(+), 1 deletion(-)

diff --git a/lxd/cgroup/abstraction.go b/lxd/cgroup/abstraction.go
index 61b7ccf664..20d77d9306 100644
--- a/lxd/cgroup/abstraction.go
+++ b/lxd/cgroup/abstraction.go
@@ -243,7 +243,6 @@ func (cg *CGroup) SetBlkioWeight(value string) error {
 		return ErrControllerMissing
 	}
 	return ErrUnknownVersion
-
 }
 
 // SetCPUShare sets the weight of each group in the same hierarchy
@@ -323,3 +322,47 @@ func (cg *CGroup) SetMaxHugepages(pageType string, value string) error {
 	}
 	return ErrUnknownVersion
 }
+
+// GetEffectiveCpuset returns the current set of CPUs for the cgroup
+func (cg *CGroup) GetEffectiveCpuset() (string, error) {
+	// Confirm we have the controller
+	version := cgControllers["cpuset"]
+	switch version {
+	case Unavailable:
+		return "", ErrControllerMissing
+	case V1:
+		return cg.rw.Get(version, "cpuset", "cpuset.effective_cpus")
+	case V2:
+		return cg.rw.Get(version, "cpuset", "cpuset.cpus.effective")
+	}
+	return "", ErrUnknownVersion
+}
+
+// GetCpuset returns the current set of CPUs for the cgroup
+func (cg *CGroup) GetCpuset() (string, error) {
+	// Confirm we have the controller
+	version := cgControllers["cpuset"]
+	switch version {
+	case Unavailable:
+		return "", ErrControllerMissing
+	case V1:
+		return cg.rw.Get(version, "cpuset", "cpuset.cpus")
+	case V2:
+		return cg.rw.Get(version, "cpuset", "cpuset.cpus")
+	}
+	return "", ErrUnknownVersion
+}
+
+// SetCpuset set the currently allowed set of CPUs for the cgroups
+func (cg *CGroup) SetCpuset(value string) error {
+	version := cgControllers["cpuset"]
+	switch version {
+	case Unavailable:
+		return ErrControllerMissing
+	case V1:
+		return cg.rw.Set(version, "cpuset", "cpuset.cpus", value)
+	case V2:
+		return cg.rw.Set(version, "cpuset", "cpuset.cpus", value)
+	}
+	return ErrUnknownVersion
+}

From 57f08497dc890f8ef681190b680efabec1abce03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 6 Nov 2020 23:44:09 -0500
Subject: [PATCH 4/8] lxd/cgroup: Fix warning wording

---
 lxd/cgroup/init.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lxd/cgroup/init.go b/lxd/cgroup/init.go
index 7338e6d47e..26c9163fdd 100644
--- a/lxd/cgroup/init.go
+++ b/lxd/cgroup/init.go
@@ -246,11 +246,11 @@ func (info *Info) Log() {
 	logger.Infof(" - cgroup layout: %s", info.Mode())
 
 	if !info.Supports(Blkio, nil) {
-		logger.Warnf(" - Couldn't find the CGroup blkio, I/O limits will be ignored")
+		logger.Warnf(" - Couldn't find the CGroup blkio, disk I/O limits will be ignored")
 	}
 
 	if !info.Supports(BlkioWeight, nil) {
-		logger.Warnf(" - Couldn't find the CGroup blkio.weight, I/O weight limits will be ignored")
+		logger.Warnf(" - Couldn't find the CGroup blkio.weight, disk priority will be ignored")
 	}
 
 	if !info.Supports(CPU, nil) {
@@ -282,7 +282,7 @@ func (info *Info) Log() {
 	}
 
 	if !info.Supports(NetPrio, nil) {
-		logger.Warnf(" - Couldn't find the CGroup network class controller, network limits will be ignored")
+		logger.Warnf(" - Couldn't find the CGroup network priority controller, network priority will be ignored")
 	}
 
 	if !info.Supports(Pids, nil) {

From f3b1da47bb4ccab6afee795889e1b5f33c0e33a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 6 Nov 2020 18:14:17 -0500
Subject: [PATCH 5/8] lxd/devices: Drop old workaround
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/devices.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lxd/devices.go b/lxd/devices.go
index 982b24d752..1ec90d105f 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -357,11 +357,6 @@ func deviceTaskBalance(s *state.State) {
 	}
 
 	effectiveCpus = strings.Join(effectiveCpusSlice, ",")
-
-	err = cGroupSet("cpuset", "/lxc", "cpuset.cpus", effectiveCpus)
-	if err != nil && shared.PathExists("/sys/fs/cgroup/cpuset/lxc") {
-		logger.Warn("Error setting lxd's cpuset.cpus", log.Ctx{"err": err})
-	}
 	cpus, err := resources.ParseCpuset(effectiveCpus)
 	if err != nil {
 		logger.Error("Error parsing host's cpu set", log.Ctx{"cpuset": effectiveCpus, "err": err})

From 995bd38b282b8f4770cfc87834d700c237822668 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 6 Nov 2020 18:14:25 -0500
Subject: [PATCH 6/8] lxd/devices: Port to cgroup package
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/cgroup.go  | 61 --------------------------------------------------
 lxd/devices.go | 10 +++++++--
 2 files changed, 8 insertions(+), 63 deletions(-)
 delete mode 100644 lxd/cgroup.go

diff --git a/lxd/cgroup.go b/lxd/cgroup.go
deleted file mode 100644
index 33f93cc9e8..0000000000
--- a/lxd/cgroup.go
+++ /dev/null
@@ -1,61 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"io/ioutil"
-	"os"
-	"path"
-	"strings"
-)
-
-func getInitCgroupPath(controller string) string {
-	f, err := os.Open("/proc/1/cgroup")
-	if err != nil {
-		return "/"
-	}
-	defer f.Close()
-
-	scan := bufio.NewScanner(f)
-	for scan.Scan() {
-		line := scan.Text()
-
-		fields := strings.Split(line, ":")
-		if len(fields) != 3 {
-			return "/"
-		}
-
-		if fields[2] != controller {
-			continue
-		}
-
-		initPath := string(fields[3])
-
-		// ignore trailing /init.scope if it is there
-		dir, file := path.Split(initPath)
-		if file == "init.scope" {
-			return dir
-		} else {
-			return initPath
-		}
-	}
-
-	return "/"
-}
-
-func cGroupGet(controller, cgroup, file string) (string, error) {
-	initPath := getInitCgroupPath(controller)
-	path := path.Join("/sys/fs/cgroup", controller, initPath, cgroup, file)
-
-	contents, err := ioutil.ReadFile(path)
-	if err != nil {
-		return "", err
-	}
-	return strings.Trim(string(contents), "\n"), nil
-}
-
-func cGroupSet(controller, cgroup, file string, value string) error {
-	initPath := getInitCgroupPath(controller)
-	path := path.Join("/sys/fs/cgroup", controller, initPath, cgroup, file)
-
-	return ioutil.WriteFile(path, []byte(value), 0755)
-}
diff --git a/lxd/devices.go b/lxd/devices.go
index 1ec90d105f..a30a8a244f 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -330,10 +330,16 @@ func deviceTaskBalance(s *state.State) {
 	}
 
 	// Get effective cpus list - those are all guaranteed to be online
-	effectiveCpus, err := cGroupGet("cpuset", "/", "cpuset.effective_cpus")
+	cg, err := cgroup.NewFileReadWriter(1, true)
+	if err != nil {
+		logger.Errorf("Unable to load cgroup writer: %v", err)
+		return
+	}
+
+	effectiveCpus, err := cg.GetEffectiveCpuset()
 	if err != nil {
 		// Older kernel - use cpuset.cpus
-		effectiveCpus, err = cGroupGet("cpuset", "/", "cpuset.cpus")
+		effectiveCpus, err = cg.GetCpuset()
 		if err != nil {
 			logger.Errorf("Error reading host's cpuset.cpus")
 			return

From c8e408eeb9d77405cfdf7828abbebdfe38e72b6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 6 Nov 2020 23:28:56 -0500
Subject: [PATCH 7/8] lxd/instance: Replace CGroupGet/CGroupSet
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/instance/drivers/driver_lxc.go  | 4 ++++
 lxd/instance/drivers/driver_qemu.go | 5 +++--
 lxd/instance/instance_interface.go  | 3 ++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/lxd/instance/drivers/driver_lxc.go b/lxd/instance/drivers/driver_lxc.go
index 50404c02ef..4af9b98cfa 100644
--- a/lxd/instance/drivers/driver_lxc.go
+++ b/lxd/instance/drivers/driver_lxc.go
@@ -7050,6 +7050,10 @@ func (c *lxc) maasDelete() error {
 	return c.state.MAAS.DeleteContainer(c)
 }
 
+func (c *lxc) CGroup() (*cgroup.CGroup, error) {
+	return c.cgroup(nil)
+}
+
 func (c *lxc) cgroup(cc *liblxc.Container) (*cgroup.CGroup, error) {
 	rw := lxcCgroupReadWriter{}
 	if cc != nil {
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 41174d770e..bda16f90ae 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -29,6 +29,7 @@ import (
 	lxdClient "github.com/lxc/lxd/client"
 	"github.com/lxc/lxd/lxd/apparmor"
 	"github.com/lxc/lxd/lxd/backup"
+	"github.com/lxc/lxd/lxd/cgroup"
 	"github.com/lxc/lxd/lxd/cluster"
 	"github.com/lxc/lxd/lxd/db"
 	"github.com/lxc/lxd/lxd/db/query"
@@ -3934,8 +3935,8 @@ func (vm *qemu) Migrate(args *instance.CriuMigrationArgs) error {
 }
 
 // CGroupSet is not implemented for VMs.
-func (vm *qemu) CGroupSet(key string, value string) error {
-	return instance.ErrNotImplemented
+func (vm *qemu) CGroup() (*cgroup.CGroup, error) {
+	return nil, instance.ErrNotImplemented
 }
 
 // VolatileSet sets one or more volatile config keys.
diff --git a/lxd/instance/instance_interface.go b/lxd/instance/instance_interface.go
index 89f628e87a..3992870f35 100644
--- a/lxd/instance/instance_interface.go
+++ b/lxd/instance/instance_interface.go
@@ -8,6 +8,7 @@ import (
 	liblxc "gopkg.in/lxc/go-lxc.v2"
 
 	"github.com/lxc/lxd/lxd/backup"
+	"github.com/lxc/lxd/lxd/cgroup"
 	"github.com/lxc/lxd/lxd/db"
 	deviceConfig "github.com/lxc/lxd/lxd/device/config"
 	"github.com/lxc/lxd/lxd/instance/instancetype"
@@ -75,7 +76,7 @@ type Instance interface {
 	DevPaths() []string
 
 	// Live configuration.
-	CGroupSet(key string, value string) error
+	CGroup() (*cgroup.CGroup, error)
 	VolatileSet(changes map[string]string) error
 
 	// File handling.

From 4d982b749e8fd3fca3bcb2d358b3eb207b1ebfb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 6 Nov 2020 23:29:46 -0500
Subject: [PATCH 8/8] lxd/devices: Update to use cgroup abstraction
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/devices.go | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/lxd/devices.go b/lxd/devices.go
index a30a8a244f..e72e125fc9 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -479,7 +479,13 @@ func deviceTaskBalance(s *state.State) {
 		}
 
 		sort.Strings(set)
-		err := ctn.CGroupSet("cpuset.cpus", strings.Join(set, ","))
+		cg, err := ctn.CGroup()
+		if err != nil {
+			logger.Error("balance: Unable to get cgroup struct", log.Ctx{"name": ctn.Name(), "err": err, "value": strings.Join(set, ",")})
+			continue
+		}
+
+		err = cg.SetCpuset(strings.Join(set, ","))
 		if err != nil {
 			logger.Error("balance: Unable to set cpuset", log.Ctx{"name": ctn.Name(), "err": err, "value": strings.Join(set, ",")})
 		}
@@ -510,7 +516,12 @@ func deviceNetworkPriority(s *state.State, netif string) {
 		}
 
 		// Set the value for the new interface
-		c.CGroupSet("net_prio.ifpriomap", fmt.Sprintf("%s %d", netif, networkInt))
+		cg, err := c.CGroup()
+		if err != nil {
+			continue
+		}
+
+		cg.SetNetIfPrio(fmt.Sprintf("%s %d", netif, networkInt))
 	}
 
 	return


More information about the lxc-devel mailing list