[lxc-devel] [lxd/master] DRAFT: CGroup OS Structure and MemoryCurrent property
shreyassood on Github
lxc-bot at linuxcontainers.org
Mon Dec 9 19:01:49 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20191209/732eb617/attachment-0001.bin>
-------------- next part --------------
From 76cedaed7e038e494973ce16c490e7641dfbb508 Mon Sep 17 00:00:00 2001
From: Shreyas Sood <shreyassood at gmail.com>
Date: Sun, 8 Dec 2019 19:49:39 -0600
Subject: [PATCH 1/6] lxd/cgroup: Initial abstraction for cgroupsv2 with pids
support
Signed-off-by: Shreyas Sood <shreyassood at gmail.com>
---
lxd/cgroup/cgroup.go | 69 ++++++++++++++++++
lxd/container_lxc.go | 108 ++++++++++++++++++++---------
lxd/devices.go | 4 +-
lxd/instance/instance_interface.go | 7 +-
lxd/instance/qemu/vm_qemu.go | 17 ++++-
5 files changed, 165 insertions(+), 40 deletions(-)
create mode 100644 lxd/cgroup/cgroup.go
diff --git a/lxd/cgroup/cgroup.go b/lxd/cgroup/cgroup.go
new file mode 100644
index 0000000000..756d9d6c11
--- /dev/null
+++ b/lxd/cgroup/cgroup.go
@@ -0,0 +1,69 @@
+package cgroup
+
+import (
+ "fmt"
+ "gopkg.in/lxc/go-lxc.v2"
+)
+
+type Property int
+const (
+ PidsCurrent Property = iota
+ PidsMax
+)
+
+type configItem struct {
+ Key string
+ Value string
+}
+
+// Get finds property values on a lxcContainer
+func Get(c *lxc.Container, property Property) ([]string, error) {
+ switch property {
+
+ // Properties which have the same functionality for both v1 and v2
+ case PidsCurrent:
+ return c.CgroupItem("pids.current"), nil
+ }
+
+ return nil, fmt.Errorf("CGroup Property not supported for Get")
+}
+
+// Set sets a property on a lxcContainer
+func Set(c *lxc.Container, property Property, value string) error {
+
+ configs, e := SetConfigMap(property, value)
+ if e != nil {
+ return e
+ }
+
+ for _, rule := range configs {
+ err := c.SetCgroupItem(rule.Key, rule.Value)
+
+ if err != nil {
+ return fmt.Errorf("Failure while trying to set property: %s", err)
+ }
+ }
+
+ return nil
+}
+
+// SetConfigMap returns different cgroup configs to set a particular property
+func SetConfigMap(property Property, value string) ([]configItem, error) {
+
+ switch property {
+
+ // Properties which have the same functionality for both v1 and v2
+ case PidsCurrent:
+ return []configItem{
+ {Key: "pids.current", Value: value},
+ }, nil
+
+ case PidsMax:
+ return []configItem{
+ {Key: "pids.max", Value: value},
+ }, nil
+
+ }
+
+ return nil, fmt.Errorf("CGroup Property not supported for Set")
+}
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 4c6bb2999d..c9a39492c5 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -1405,7 +1405,7 @@ func (c *containerLXC) deviceAddCgroupRules(cgroups []deviceConfig.RunConfigItem
}
// Add the new device cgroup rule.
- err := c.CGroupSet(rule.Key, rule.Value)
+ err := c.CGroupSetV1(rule.Key, rule.Value)
if err != nil {
return fmt.Errorf("Failed to add cgroup rule for device")
}
@@ -2592,7 +2592,7 @@ func (c *containerLXC) Stop(stateful bool) error {
// Fork-bomb mitigation, prevent forking from this point on
if c.state.OS.CGroupPidsController {
// Attempt to disable forking new processes
- c.CGroupSet("pids.max", "0")
+ c.CGroupSet(cgroup.PidsMax, "0")
} else if c.state.OS.CGroupFreezerController {
// Attempt to freeze the container
freezer := make(chan bool, 1)
@@ -3769,7 +3769,28 @@ func (c *containerLXC) Rename(newName string) error {
return nil
}
-func (c *containerLXC) CGroupGet(key string) (string, error) {
+// CGroupGet gets the value of a supported property on a hybrid v1/v2 controller system
+func (c *containerLXC) CGroupGet(property cgroup.Property) (string, error) {
+ // Load the go-lxc struct
+ err := c.initLXC(false)
+ if err != nil {
+ return "", err
+ }
+
+ // Make sure the container is running
+ if !c.IsRunning() {
+ return "", fmt.Errorf("Can't get cgroups on a stopped container")
+ }
+
+ value, err := cgroup.Get(c.c, property)
+
+ if err != nil {
+ return "", err
+ }
+ return strings.Join(value, "\n"), nil
+}
+
+func (c *containerLXC) CGroupGetV1(key string) (string, error) {
// Load the go-lxc struct
err := c.initLXC(false)
if err != nil {
@@ -3785,7 +3806,28 @@ func (c *containerLXC) CGroupGet(key string) (string, error) {
return strings.Join(value, "\n"), nil
}
-func (c *containerLXC) CGroupSet(key string, value string) error {
+// CGroupSet sets a supported property on a hybrid v1/v2 controller system
+func (c *containerLXC) CGroupSet(property cgroup.Property, value string) error {
+ // Load the go-lxc struct
+ err := c.initLXC(false)
+ if err != nil {
+ return err
+ }
+
+ // Make sure the container is running
+ if !c.IsRunning() {
+ return fmt.Errorf("Can't set cgroups on a stopped container")
+ }
+
+ err = cgroup.Set(c.c, property, value)
+ if err != nil {
+ return fmt.Errorf("Failed to set cgroup item %d=\"%s\": %s", property, value, err)
+ }
+
+ return nil
+}
+
+func (c *containerLXC) CGroupSetV1(key string, value string) error {
// Load the go-lxc struct
err := c.initLXC(false)
if err != nil {
@@ -4208,7 +4250,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
priority = 10
}
- err = c.CGroupSet("blkio.weight", fmt.Sprintf("%d", priority))
+ err = c.CGroupSetV1("blkio.weight", fmt.Sprintf("%d", priority))
if err != nil {
return err
}
@@ -4249,52 +4291,52 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
// Store the old values for revert
oldMemswLimit := ""
if c.state.OS.CGroupSwapAccounting {
- oldMemswLimit, err = c.CGroupGet("memory.memsw.limit_in_bytes")
+ oldMemswLimit, err = c.CGroupGetV1("memory.memsw.limit_in_bytes")
if err != nil {
oldMemswLimit = ""
}
}
- oldLimit, err := c.CGroupGet("memory.limit_in_bytes")
+ oldLimit, err := c.CGroupGetV1("memory.limit_in_bytes")
if err != nil {
oldLimit = ""
}
- oldSoftLimit, err := c.CGroupGet("memory.soft_limit_in_bytes")
+ oldSoftLimit, err := c.CGroupGetV1("memory.soft_limit_in_bytes")
if err != nil {
oldSoftLimit = ""
}
revertMemory := func() {
if oldSoftLimit != "" {
- c.CGroupSet("memory.soft_limit_in_bytes", oldSoftLimit)
+ c.CGroupSetV1("memory.soft_limit_in_bytes", oldSoftLimit)
}
if oldLimit != "" {
- c.CGroupSet("memory.limit_in_bytes", oldLimit)
+ c.CGroupSetV1("memory.limit_in_bytes", oldLimit)
}
if oldMemswLimit != "" {
- c.CGroupSet("memory.memsw.limit_in_bytes", oldMemswLimit)
+ c.CGroupSetV1("memory.memsw.limit_in_bytes", oldMemswLimit)
}
}
// Reset everything
if c.state.OS.CGroupSwapAccounting {
- err = c.CGroupSet("memory.memsw.limit_in_bytes", "-1")
+ err = c.CGroupSetV1("memory.memsw.limit_in_bytes", "-1")
if err != nil {
revertMemory()
return err
}
}
- err = c.CGroupSet("memory.limit_in_bytes", "-1")
+ err = c.CGroupSetV1("memory.limit_in_bytes", "-1")
if err != nil {
revertMemory()
return err
}
- err = c.CGroupSet("memory.soft_limit_in_bytes", "-1")
+ err = c.CGroupSetV1("memory.soft_limit_in_bytes", "-1")
if err != nil {
revertMemory()
return err
@@ -4303,26 +4345,26 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
// Set the new values
if memoryEnforce == "soft" {
// Set new limit
- err = c.CGroupSet("memory.soft_limit_in_bytes", memory)
+ err = c.CGroupSetV1("memory.soft_limit_in_bytes", memory)
if err != nil {
revertMemory()
return err
}
} else {
if c.state.OS.CGroupSwapAccounting && (memorySwap == "" || shared.IsTrue(memorySwap)) {
- err = c.CGroupSet("memory.limit_in_bytes", memory)
+ err = c.CGroupSetV1("memory.limit_in_bytes", memory)
if err != nil {
revertMemory()
return err
}
- err = c.CGroupSet("memory.memsw.limit_in_bytes", memory)
+ err = c.CGroupSetV1("memory.memsw.limit_in_bytes", memory)
if err != nil {
revertMemory()
return err
}
} else {
- err = c.CGroupSet("memory.limit_in_bytes", memory)
+ err = c.CGroupSetV1("memory.limit_in_bytes", memory)
if err != nil {
revertMemory()
return err
@@ -4336,7 +4378,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
return err
}
- err = c.CGroupSet("memory.soft_limit_in_bytes", fmt.Sprintf("%.0f", float64(valueInt)*0.9))
+ err = c.CGroupSetV1("memory.soft_limit_in_bytes", fmt.Sprintf("%.0f", float64(valueInt)*0.9))
if err != nil {
revertMemory()
return err
@@ -4348,7 +4390,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
memorySwap := c.expandedConfig["limits.memory.swap"]
memorySwapPriority := c.expandedConfig["limits.memory.swap.priority"]
if memorySwap != "" && !shared.IsTrue(memorySwap) {
- err = c.CGroupSet("memory.swappiness", "0")
+ err = c.CGroupSetV1("memory.swappiness", "0")
if err != nil {
return err
}
@@ -4361,7 +4403,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
}
}
- err = c.CGroupSet("memory.swappiness", fmt.Sprintf("%d", 60-10+priority))
+ err = c.CGroupSetV1("memory.swappiness", fmt.Sprintf("%d", 60-10+priority))
if err != nil {
return err
}
@@ -4387,17 +4429,17 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
return err
}
- err = c.CGroupSet("cpu.shares", cpuShares)
+ err = c.CGroupSetV1("cpu.shares", cpuShares)
if err != nil {
return err
}
- err = c.CGroupSet("cpu.cfs_period_us", cpuCfsPeriod)
+ err = c.CGroupSetV1("cpu.cfs_period_us", cpuCfsPeriod)
if err != nil {
return err
}
- err = c.CGroupSet("cpu.cfs_quota_us", cpuCfsQuota)
+ err = c.CGroupSetV1("cpu.cfs_quota_us", cpuCfsQuota)
if err != nil {
return err
}
@@ -4407,7 +4449,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
}
if value == "" {
- err = c.CGroupSet("pids.max", "max")
+ err = c.CGroupSet(cgroup.PidsCurrent, "max")
if err != nil {
return err
}
@@ -4417,7 +4459,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
return err
}
- err = c.CGroupSet("pids.max", fmt.Sprintf("%d", valueInt))
+ err = c.CGroupSet(cgroup.PidsMax, fmt.Sprintf("%d", valueInt))
if err != nil {
return err
}
@@ -5732,7 +5774,7 @@ func (c *containerLXC) cpuState() api.InstanceStateCPU {
}
// CPU usage in seconds
- value, err := c.CGroupGet("cpuacct.usage")
+ value, err := c.CGroupGetV1("cpuacct.usage")
if err != nil {
cpu.Usage = -1
return cpu
@@ -5803,14 +5845,14 @@ func (c *containerLXC) memoryState() api.InstanceStateMemory {
}
// Memory in bytes
- value, err := c.CGroupGet("memory.usage_in_bytes")
+ value, err := c.CGroupGetV1("memory.usage_in_bytes")
valueInt, err1 := strconv.ParseInt(value, 10, 64)
if err == nil && err1 == nil {
memory.Usage = valueInt
}
// Memory peak in bytes
- value, err = c.CGroupGet("memory.max_usage_in_bytes")
+ value, err = c.CGroupGetV1("memory.max_usage_in_bytes")
valueInt, err1 = strconv.ParseInt(value, 10, 64)
if err == nil && err1 == nil {
memory.UsagePeak = valueInt
@@ -5819,7 +5861,7 @@ func (c *containerLXC) memoryState() api.InstanceStateMemory {
if c.state.OS.CGroupSwapAccounting {
// Swap in bytes
if memory.Usage > 0 {
- value, err := c.CGroupGet("memory.memsw.usage_in_bytes")
+ value, err := c.CGroupGetV1("memory.memsw.usage_in_bytes")
valueInt, err1 := strconv.ParseInt(value, 10, 64)
if err == nil && err1 == nil {
memory.SwapUsage = valueInt - memory.Usage
@@ -5828,7 +5870,7 @@ func (c *containerLXC) memoryState() api.InstanceStateMemory {
// Swap peak in bytes
if memory.UsagePeak > 0 {
- value, err = c.CGroupGet("memory.memsw.max_usage_in_bytes")
+ value, err = c.CGroupGetV1("memory.memsw.max_usage_in_bytes")
valueInt, err1 = strconv.ParseInt(value, 10, 64)
if err == nil && err1 == nil {
memory.SwapUsagePeak = valueInt - memory.UsagePeak
@@ -5905,7 +5947,7 @@ func (c *containerLXC) processesState() int64 {
}
if c.state.OS.CGroupPidsController {
- value, err := c.CGroupGet("pids.current")
+ value, err := c.CGroupGet(cgroup.PidsCurrent)
if err != nil {
return -1
}
@@ -6527,7 +6569,7 @@ func (c *containerLXC) setNetworkPriority() error {
success := false
var last_error error
for _, netif := range netifs {
- err = c.CGroupSet("net_prio.ifpriomap", fmt.Sprintf("%s %d", netif.Name, networkInt))
+ err = c.CGroupSetV1("net_prio.ifpriomap", fmt.Sprintf("%s %d", netif.Name, networkInt))
if err == nil {
success = true
} else {
diff --git a/lxd/devices.go b/lxd/devices.go
index fb8a84f733..28d4d3a34d 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -400,7 +400,7 @@ func deviceTaskBalance(s *state.State) {
}
sort.Strings(set)
- err := ctn.CGroupSet("cpuset.cpus", strings.Join(set, ","))
+ err := ctn.CGroupSetV1("cpuset.cpus", strings.Join(set, ","))
if err != nil {
logger.Error("balance: Unable to set cpuset", log.Ctx{"name": ctn.Name(), "err": err, "value": strings.Join(set, ",")})
}
@@ -431,7 +431,7 @@ func deviceNetworkPriority(s *state.State, netif string) {
}
// Set the value for the new interface
- c.CGroupSet("net_prio.ifpriomap", fmt.Sprintf("%s %d", netif, networkInt))
+ c.CGroupSetV1("net_prio.ifpriomap", fmt.Sprintf("%s %d", netif, networkInt))
}
return
diff --git a/lxd/instance/instance_interface.go b/lxd/instance/instance_interface.go
index 9c56b0ddfe..22facc7540 100644
--- a/lxd/instance/instance_interface.go
+++ b/lxd/instance/instance_interface.go
@@ -6,6 +6,7 @@ import (
"time"
"github.com/lxc/lxd/lxd/backup"
+ "github.com/lxc/lxd/lxd/cgroup"
"github.com/lxc/lxd/lxd/db"
deviceConfig "github.com/lxc/lxd/lxd/device/config"
"github.com/lxc/lxd/lxd/instance/instancetype"
@@ -40,8 +41,10 @@ type Instance interface {
Export(w io.Writer, properties map[string]string) error
// Live configuration
- CGroupGet(key string) (string, error)
- CGroupSet(key string, value string) error
+ CGroupGet(property cgroup.Property) (string, error)
+ CGroupGetV1(key string) (string, error)
+ CGroupSet(property cgroup.Property, value string) error
+ CGroupSetV1(key string, value string) error
VolatileSet(changes map[string]string) error
// File handling
diff --git a/lxd/instance/qemu/vm_qemu.go b/lxd/instance/qemu/vm_qemu.go
index 5f149794b2..3d456201f2 100644
--- a/lxd/instance/qemu/vm_qemu.go
+++ b/lxd/instance/qemu/vm_qemu.go
@@ -23,6 +23,7 @@ import (
lxdClient "github.com/lxc/lxd/client"
"github.com/lxc/lxd/lxd/backup"
+ "github.com/lxc/lxd/lxd/cgroup"
"github.com/lxc/lxd/lxd/cluster"
"github.com/lxc/lxd/lxd/db"
"github.com/lxc/lxd/lxd/db/query"
@@ -2208,16 +2209,26 @@ func (vm *Qemu) Export(w io.Writer, properties map[string]string) error {
return fmt.Errorf("Export Not implemented")
}
-// CGroupGet is not implemented for VMs.
-func (vm *Qemu) CGroupGet(key string) (string, error) {
+// CGroupGetV1 is not implemented for VMs.
+func (vm *Qemu) CGroupGet(property cgroup.Property) (string, error) {
return "", fmt.Errorf("CGroupGet Not implemented")
}
+// CGroupGetV1 is not implemented for VMs.
+func (vm *Qemu) CGroupGetV1(key string) (string, error) {
+ return "", fmt.Errorf("CGroupGetV1 Not implemented")
+}
+
// CGroupSet is not implemented for VMs.
-func (vm *Qemu) CGroupSet(key string, value string) error {
+func (vm *Qemu) CGroupSet(property cgroup.Property, value string) error {
return fmt.Errorf("CGroupSet Not implemented")
}
+// CGroupSetV1 is not implemented for VMs.
+func (vm *Qemu) CGroupSetV1(key string, value string) error {
+ return fmt.Errorf("CGroupSetV1 Not implemented")
+}
+
// VolatileSet sets one or more volatile config keys.
func (vm *Qemu) VolatileSet(changes map[string]string) error {
// Sanity check.
From 6c8df05d88355b8e8999a27dbb45a884d595c578 Mon Sep 17 00:00:00 2001
From: Lucinda Nguyen <lucinda.onguyen at gmail.com>
Date: Sun, 8 Dec 2019 21:50:44 -0600
Subject: [PATCH 2/6] OS structure modified to detect v1/v2 controllers
---
lxd/sys/cgroup.go | 57 +++++++++++++++++++++++++++++++++++++++++++----
lxd/sys/os.go | 8 +++++++
2 files changed, 61 insertions(+), 4 deletions(-)
diff --git a/lxd/sys/cgroup.go b/lxd/sys/cgroup.go
index 9faaf7b775..b16dc7b744 100644
--- a/lxd/sys/cgroup.go
+++ b/lxd/sys/cgroup.go
@@ -4,9 +4,12 @@ package sys
import (
"fmt"
-
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/logger"
+ "os"
+ "path"
+ "bufio"
+ "strings"
)
// Detect CGroup support.
@@ -24,8 +27,54 @@ func (s *OS) initCGroup() {
&s.CGroupPidsController,
&s.CGroupSwapAccounting,
}
- for i, flag := range flags {
- *flag = shared.PathExists("/sys/fs/cgroup/" + cGroups[i].path)
+
+ flags_v2 := []*bool{
+ //tell us if it's a v1 or v2 controller
+ &s.CGroupMemoryControllerV2,
+ &s.CGroupCPUControllerV2,
+ &s.CGroupPidsControllerV2,
+ }
+ var j int = 0
+ for i,flag := range flags {
+ if cGroups[i].path == "memory" || cGroups[i].path == "pids" || cGroups[i].path == "cpu" {
+ //have to check v1 and v2
+ //v1
+ *flag = shared.PathExists("/sys/fs/cgroup/" + cGroups[i].path)
+ *flags_v2[j] = false
+ j++
+ //then set flag for flags_v2 to be false
+ //v2
+ if !*flag {
+ //read this file to check which controllers are supported for v2
+ path := path.Join("/sys/fs/cgroup", "unified", "cgroup.controllers")
+ file, err := os.Open(path)
+ if err != nil {
+ logger.Debugf("Can't open file")
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ line_controllers:= scanner.Text()
+ //if controller is within file, version will be 2
+ if strings.Contains(line_controllers, cGroups[i].path) {
+ *flag = true
+ *flags_v2[j] = true
+ j++
+ }
+
+ }
+
+ if err := scanner.Err(); err != nil {
+ logger.Debugf("Can't something")
+ }
+
+ }
+ } else {
+
+ *flag = shared.PathExists("/sys/fs/cgroup/" + cGroups[i].path)
+ }
+
if !*flag {
logger.Warnf(cGroups[i].warn)
}
@@ -55,4 +104,4 @@ var cGroups = []struct {
{"net_prio", cGroupMissing("network class controller", "network limits will be ignored")},
{"pids", cGroupMissing("pids controller", "process limits will be ignored")},
{"memory/memory.memsw.limit_in_bytes", cGroupDisabled("memory swap accounting", "swap limits will be ignored")},
-}
+}
\ No newline at end of file
diff --git a/lxd/sys/os.go b/lxd/sys/os.go
index 976522a788..c69565789d 100644
--- a/lxd/sys/os.go
+++ b/lxd/sys/os.go
@@ -68,6 +68,14 @@ type OS struct {
CGroupPidsController bool
CGroupSwapAccounting bool
+ //detect if controller is v1 or v2
+ CGroupMemoryControllerV2 bool
+ CGroupCPUControllerV2 bool
+ CGroupPidsControllerV2 bool
+
+ CGroupUnifiedHierarchy bool
+
+
// Kernel features
NetnsGetifaddrs bool
SeccompListener bool
From 1b9ef297e7d091e773b46acc58ef73ebce6d78c7 Mon Sep 17 00:00:00 2001
From: lucindan <44307632+lucindan at users.noreply.github.com>
Date: Sun, 8 Dec 2019 23:41:22 -0600
Subject: [PATCH 3/6] Fixed index out of range, changed path
---
lxd/sys/cgroup.go | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/lxd/sys/cgroup.go b/lxd/sys/cgroup.go
index b16dc7b744..4304fc631c 100644
--- a/lxd/sys/cgroup.go
+++ b/lxd/sys/cgroup.go
@@ -3,13 +3,14 @@
package sys
import (
+ "bufio"
"fmt"
- "github.com/lxc/lxd/shared"
- "github.com/lxc/lxd/shared/logger"
"os"
"path"
- "bufio"
"strings"
+
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/logger"
)
// Detect CGroup support.
@@ -41,12 +42,13 @@ func (s *OS) initCGroup() {
//v1
*flag = shared.PathExists("/sys/fs/cgroup/" + cGroups[i].path)
*flags_v2[j] = false
- j++
//then set flag for flags_v2 to be false
//v2
if !*flag {
//read this file to check which controllers are supported for v2
- path := path.Join("/sys/fs/cgroup", "unified", "cgroup.controllers")
+ //path := path.Join("/sys/fs/cgroup", "unified", "cgroup.controllers")
+ //TODO-our: need to change this path for hybrid
+ path := path.Join("/sys/fs/cgroup", "cgroup.controllers")
file, err := os.Open(path)
if err != nil {
logger.Debugf("Can't open file")
@@ -60,11 +62,10 @@ func (s *OS) initCGroup() {
if strings.Contains(line_controllers, cGroups[i].path) {
*flag = true
*flags_v2[j] = true
- j++
}
}
-
+ j++
if err := scanner.Err(); err != nil {
logger.Debugf("Can't something")
}
@@ -104,4 +105,4 @@ var cGroups = []struct {
{"net_prio", cGroupMissing("network class controller", "network limits will be ignored")},
{"pids", cGroupMissing("pids controller", "process limits will be ignored")},
{"memory/memory.memsw.limit_in_bytes", cGroupDisabled("memory swap accounting", "swap limits will be ignored")},
-}
\ No newline at end of file
+}
From 92cac397a6becacde82031cb6e24ad3b4b4064be Mon Sep 17 00:00:00 2001
From: lucindan <44307632+lucindan at users.noreply.github.com>
Date: Mon, 9 Dec 2019 00:12:51 -0600
Subject: [PATCH 4/6] fixed index
---
lxd/sys/cgroup.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lxd/sys/cgroup.go b/lxd/sys/cgroup.go
index 4304fc631c..cfd1692f8c 100644
--- a/lxd/sys/cgroup.go
+++ b/lxd/sys/cgroup.go
@@ -65,12 +65,13 @@ func (s *OS) initCGroup() {
}
}
- j++
+
if err := scanner.Err(); err != nil {
logger.Debugf("Can't something")
}
}
+ j++
} else {
*flag = shared.PathExists("/sys/fs/cgroup/" + cGroups[i].path)
From 8c2af4567a7786cb33a026cea2d30981860a3ef0 Mon Sep 17 00:00:00 2001
From: Shreyas Sood <shreyassood at gmail.com>
Date: Mon, 9 Dec 2019 02:51:17 -0600
Subject: [PATCH 5/6] Clean CGroup struct code
---
lxd/container_lxc.go | 45 +++++++++++++++--------------
lxd/container_state.go | 5 ++--
lxd/device/disk.go | 5 ++--
lxd/devices.go | 11 +++----
lxd/sys/cgroup.go | 65 +++++++++++-------------------------------
lxd/sys/os.go | 38 +++++++++++++-----------
6 files changed, 72 insertions(+), 97 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index c9a39492c5..b7ed164b35 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -43,6 +43,7 @@ import (
"github.com/lxc/lxd/lxd/state"
storagePools "github.com/lxc/lxd/lxd/storage"
storageDrivers "github.com/lxc/lxd/lxd/storage/drivers"
+ "github.com/lxc/lxd/lxd/sys"
"github.com/lxc/lxd/lxd/template"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
@@ -845,7 +846,7 @@ func (c *containerLXC) initLXC(config bool) error {
}
// Configure devices cgroup
- if c.IsPrivileged() && !c.state.OS.RunningInUserNS && c.state.OS.CGroupDevicesController {
+ if c.IsPrivileged() && !c.state.OS.RunningInUserNS && c.state.OS.CGroupDevicesController != sys.CGroupDisabled {
err = lxcSetConfigItem(cc, "lxc.cgroup.devices.deny", "a")
if err != nil {
return err
@@ -1080,7 +1081,7 @@ func (c *containerLXC) initLXC(config bool) error {
}
// Memory limits
- if c.state.OS.CGroupMemoryController {
+ if c.state.OS.CGroupMemoryController != sys.CGroupDisabled {
memory := c.expandedConfig["limits.memory"]
memoryEnforce := c.expandedConfig["limits.memory.enforce"]
memorySwap := c.expandedConfig["limits.memory.swap"]
@@ -1114,7 +1115,7 @@ func (c *containerLXC) initLXC(config bool) error {
return err
}
} else {
- if c.state.OS.CGroupSwapAccounting && (memorySwap == "" || shared.IsTrue(memorySwap)) {
+ if c.state.OS.CGroupSwapAccounting == sys.CGroupDisabled && (memorySwap == "" || shared.IsTrue(memorySwap)) {
err = lxcSetConfigItem(cc, "lxc.cgroup.memory.limit_in_bytes", fmt.Sprintf("%d", valueInt))
if err != nil {
return err
@@ -1160,7 +1161,7 @@ func (c *containerLXC) initLXC(config bool) error {
cpuPriority := c.expandedConfig["limits.cpu.priority"]
cpuAllowance := c.expandedConfig["limits.cpu.allowance"]
- if (cpuPriority != "" || cpuAllowance != "") && c.state.OS.CGroupCPUController {
+ if (cpuPriority != "" || cpuAllowance != "") && c.state.OS.CGroupCPUController != sys.CGroupDisabled {
cpuShares, cpuCfsQuota, cpuCfsPeriod, err := cgroup.ParseCPU(cpuAllowance, cpuPriority)
if err != nil {
return err
@@ -1189,7 +1190,7 @@ func (c *containerLXC) initLXC(config bool) error {
}
// Processes
- if c.state.OS.CGroupPidsController {
+ if c.state.OS.CGroupPidsController != sys.CGroupDisabled {
processes := c.expandedConfig["limits.processes"]
if processes != "" {
valueInt, err := strconv.ParseInt(processes, 10, 64)
@@ -1400,7 +1401,7 @@ func (c *containerLXC) deviceStaticShiftMounts(mounts []deviceConfig.MountEntryI
func (c *containerLXC) deviceAddCgroupRules(cgroups []deviceConfig.RunConfigItem) error {
for _, rule := range cgroups {
// Only apply devices cgroup rules if container is running privileged and host has devices cgroup controller.
- if strings.HasPrefix(rule.Key, "devices.") && (!c.isCurrentlyPrivileged() || c.state.OS.RunningInUserNS || !c.state.OS.CGroupDevicesController) {
+ if strings.HasPrefix(rule.Key, "devices.") && (!c.isCurrentlyPrivileged() || c.state.OS.RunningInUserNS || c.state.OS.CGroupDevicesController == sys.CGroupDisabled) {
continue
}
@@ -2590,10 +2591,10 @@ func (c *containerLXC) Stop(stateful bool) error {
}
// Fork-bomb mitigation, prevent forking from this point on
- if c.state.OS.CGroupPidsController {
+ if c.state.OS.CGroupPidsController != sys.CGroupDisabled {
// Attempt to disable forking new processes
c.CGroupSet(cgroup.PidsMax, "0")
- } else if c.state.OS.CGroupFreezerController {
+ } else if c.state.OS.CGroupFreezerController != sys.CGroupDisabled {
// Attempt to freeze the container
freezer := make(chan bool, 1)
go func() {
@@ -2847,7 +2848,7 @@ func (c *containerLXC) Freeze() error {
}
// Check if the CGroup is available
- if !c.state.OS.CGroupFreezerController {
+ if c.state.OS.CGroupFreezerController == sys.CGroupDisabled {
logger.Info("Unable to freeze container (lack of kernel support)", ctxMap)
return nil
}
@@ -2895,7 +2896,7 @@ func (c *containerLXC) Unfreeze() error {
}
// Check if the CGroup is available
- if !c.state.OS.CGroupFreezerController {
+ if c.state.OS.CGroupFreezerController == sys.CGroupDisabled {
logger.Info("Unable to unfreeze container (lack of kernel support)", ctxMap)
return nil
}
@@ -4231,7 +4232,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
}
}
} else if key == "limits.disk.priority" {
- if !c.state.OS.CGroupBlkioController {
+ if c.state.OS.CGroupBlkioController == sys.CGroupDisabled {
continue
}
@@ -4256,7 +4257,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
}
} else if key == "limits.memory" || strings.HasPrefix(key, "limits.memory.") {
// Skip if no memory CGroup
- if !c.state.OS.CGroupMemoryController {
+ if c.state.OS.CGroupMemoryController == sys.CGroupDisabled {
continue
}
@@ -4290,7 +4291,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
// Store the old values for revert
oldMemswLimit := ""
- if c.state.OS.CGroupSwapAccounting {
+ if c.state.OS.CGroupSwapAccounting != sys.CGroupDisabled {
oldMemswLimit, err = c.CGroupGetV1("memory.memsw.limit_in_bytes")
if err != nil {
oldMemswLimit = ""
@@ -4322,7 +4323,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
}
// Reset everything
- if c.state.OS.CGroupSwapAccounting {
+ if c.state.OS.CGroupSwapAccounting != sys.CGroupDisabled {
err = c.CGroupSetV1("memory.memsw.limit_in_bytes", "-1")
if err != nil {
revertMemory()
@@ -4351,7 +4352,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
return err
}
} else {
- if c.state.OS.CGroupSwapAccounting && (memorySwap == "" || shared.IsTrue(memorySwap)) {
+ if c.state.OS.CGroupSwapAccounting != sys.CGroupDisabled && (memorySwap == "" || shared.IsTrue(memorySwap)) {
err = c.CGroupSetV1("memory.limit_in_bytes", memory)
if err != nil {
revertMemory()
@@ -4419,7 +4420,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
cgroup.TaskSchedulerTrigger("container", c.name, "changed")
} else if key == "limits.cpu.priority" || key == "limits.cpu.allowance" {
// Skip if no cpu CGroup
- if !c.state.OS.CGroupCPUController {
+ if c.state.OS.CGroupCPUController == sys.CGroupDisabled {
continue
}
@@ -4444,7 +4445,7 @@ func (c *containerLXC) Update(args db.InstanceArgs, userRequested bool) error {
return err
}
} else if key == "limits.processes" {
- if !c.state.OS.CGroupPidsController {
+ if c.state.OS.CGroupPidsController == sys.CGroupDisabled {
continue
}
@@ -5769,7 +5770,7 @@ func (c *containerLXC) Exec(command []string, env map[string]string, stdin *os.F
func (c *containerLXC) cpuState() api.InstanceStateCPU {
cpu := api.InstanceStateCPU{}
- if !c.state.OS.CGroupCPUacctController {
+ if c.state.OS.CGroupCPUacctController == sys.CGroupDisabled {
return cpu
}
@@ -5840,7 +5841,7 @@ func (c *containerLXC) diskState() map[string]api.InstanceStateDisk {
func (c *containerLXC) memoryState() api.InstanceStateMemory {
memory := api.InstanceStateMemory{}
- if !c.state.OS.CGroupMemoryController {
+ if c.state.OS.CGroupMemoryController == sys.CGroupDisabled {
return memory
}
@@ -5858,7 +5859,7 @@ func (c *containerLXC) memoryState() api.InstanceStateMemory {
memory.UsagePeak = valueInt
}
- if c.state.OS.CGroupSwapAccounting {
+ if c.state.OS.CGroupSwapAccounting != sys.CGroupDisabled {
// Swap in bytes
if memory.Usage > 0 {
value, err := c.CGroupGetV1("memory.memsw.usage_in_bytes")
@@ -5946,7 +5947,7 @@ func (c *containerLXC) processesState() int64 {
return 0
}
- if c.state.OS.CGroupPidsController {
+ if c.state.OS.CGroupPidsController != sys.CGroupDisabled {
value, err := c.CGroupGet(cgroup.PidsCurrent)
if err != nil {
return -1
@@ -6544,7 +6545,7 @@ func (c *containerLXC) setNetworkPriority() error {
}
// Don't bother if the cgroup controller doesn't exist
- if !c.state.OS.CGroupNetPrioController {
+ if c.state.OS.CGroupNetPrioController == sys.CGroupDisabled {
return nil
}
diff --git a/lxd/container_state.go b/lxd/container_state.go
index 2c55c022ad..addd5af44b 100644
--- a/lxd/container_state.go
+++ b/lxd/container_state.go
@@ -14,6 +14,7 @@ import (
"github.com/lxc/lxd/lxd/response"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/lxd/sys"
)
func containerState(d *Daemon, r *http.Request) response.Response {
@@ -190,7 +191,7 @@ func containerStatePut(d *Daemon, r *http.Request) response.Response {
return nil
}
case shared.Freeze:
- if !d.os.CGroupFreezerController {
+ if d.os.CGroupFreezerController == sys.CGroupDisabled {
return response.BadRequest(fmt.Errorf("This system doesn't support freezing containers"))
}
@@ -200,7 +201,7 @@ func containerStatePut(d *Daemon, r *http.Request) response.Response {
return c.Freeze()
}
case shared.Unfreeze:
- if !d.os.CGroupFreezerController {
+ if d.os.CGroupFreezerController == sys.CGroupDisabled {
return response.BadRequest(fmt.Errorf("This system doesn't support unfreezing containers"))
}
diff --git a/lxd/device/disk.go b/lxd/device/disk.go
index 1f10f2055c..45a6b3b635 100644
--- a/lxd/device/disk.go
+++ b/lxd/device/disk.go
@@ -16,6 +16,7 @@ import (
deviceConfig "github.com/lxc/lxd/lxd/device/config"
"github.com/lxc/lxd/lxd/instance/instancetype"
storagePools "github.com/lxc/lxd/lxd/storage"
+ "github.com/lxc/lxd/lxd/sys"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/logger"
@@ -468,7 +469,7 @@ func (d *disk) generateLimits(runConf *deviceConfig.RunConfig) error {
// Disk priority limits.
diskPriority := d.instance.ExpandedConfig()["limits.disk.priority"]
if diskPriority != "" {
- if d.state.OS.CGroupBlkioWeightController {
+ if d.state.OS.CGroupBlkioWeightController != sys.CGroupDisabled {
priorityInt, err := strconv.Atoi(diskPriority)
if err != nil {
return err
@@ -503,7 +504,7 @@ func (d *disk) generateLimits(runConf *deviceConfig.RunConfig) error {
}
if hasDiskLimits {
- if !d.state.OS.CGroupBlkioController {
+ if d.state.OS.CGroupBlkioController == sys.CGroupDisabled {
return fmt.Errorf("Cannot apply disk limits as blkio cgroup controller is missing")
}
diff --git a/lxd/devices.go b/lxd/devices.go
index 28d4d3a34d..f5199e6ca3 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -16,6 +16,7 @@ import (
"github.com/lxc/lxd/lxd/instance"
"github.com/lxc/lxd/lxd/instance/instancetype"
"github.com/lxc/lxd/lxd/state"
+ "github.com/lxc/lxd/lxd/sys"
"github.com/lxc/lxd/shared"
log "github.com/lxc/lxd/shared/log15"
"github.com/lxc/lxd/shared/logger"
@@ -229,7 +230,7 @@ func deviceTaskBalance(s *state.State) {
}
// Don't bother running when CGroup support isn't there
- if !s.OS.CGroupCPUsetController {
+ if s.OS.CGroupCPUsetController == sys.CGroupDisabled {
return
}
@@ -409,7 +410,7 @@ func deviceTaskBalance(s *state.State) {
func deviceNetworkPriority(s *state.State, netif string) {
// Don't bother running when CGroup support isn't there
- if !s.OS.CGroupNetPrioController {
+ if s.OS.CGroupNetPrioController == sys.CGroupDisabled {
return
}
@@ -452,7 +453,7 @@ func deviceEventListener(s *state.State) {
continue
}
- if !s.OS.CGroupCPUsetController {
+ if s.OS.CGroupCPUsetController == sys.CGroupDisabled {
continue
}
@@ -464,7 +465,7 @@ func deviceEventListener(s *state.State) {
continue
}
- if !s.OS.CGroupNetPrioController {
+ if s.OS.CGroupNetPrioController == sys.CGroupDisabled {
continue
}
@@ -479,7 +480,7 @@ func deviceEventListener(s *state.State) {
continue
}
- if !s.OS.CGroupCPUsetController {
+ if s.OS.CGroupCPUsetController == sys.CGroupDisabled {
continue
}
diff --git a/lxd/sys/cgroup.go b/lxd/sys/cgroup.go
index 4304fc631c..3f69041043 100644
--- a/lxd/sys/cgroup.go
+++ b/lxd/sys/cgroup.go
@@ -3,19 +3,18 @@
package sys
import (
- "bufio"
"fmt"
- "os"
- "path"
+ "io/ioutil"
"strings"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/logger"
)
+
// Detect CGroup support.
func (s *OS) initCGroup() {
- flags := []*bool{
+ cgroupsinfo := []*CGroupInfo{
&s.CGroupBlkioController,
&s.CGroupBlkioWeightController,
&s.CGroupCPUController,
@@ -29,54 +28,22 @@ func (s *OS) initCGroup() {
&s.CGroupSwapAccounting,
}
- flags_v2 := []*bool{
- //tell us if it's a v1 or v2 controller
- &s.CGroupMemoryControllerV2,
- &s.CGroupCPUControllerV2,
- &s.CGroupPidsControllerV2,
+ // Read all v2 controllers for later parsing
+ v2controllers := ""
+ contents, err := ioutil.ReadFile("/sys/fs/cgroup/cgroup.controllers")
+ if err != nil {
+ v2controllers = string(contents)
}
- var j int = 0
- for i,flag := range flags {
- if cGroups[i].path == "memory" || cGroups[i].path == "pids" || cGroups[i].path == "cpu" {
- //have to check v1 and v2
- //v1
- *flag = shared.PathExists("/sys/fs/cgroup/" + cGroups[i].path)
- *flags_v2[j] = false
- //then set flag for flags_v2 to be false
- //v2
- if !*flag {
- //read this file to check which controllers are supported for v2
- //path := path.Join("/sys/fs/cgroup", "unified", "cgroup.controllers")
- //TODO-our: need to change this path for hybrid
- path := path.Join("/sys/fs/cgroup", "cgroup.controllers")
- file, err := os.Open(path)
- if err != nil {
- logger.Debugf("Can't open file")
- }
- defer file.Close()
-
- scanner := bufio.NewScanner(file)
- for scanner.Scan() {
- line_controllers:= scanner.Text()
- //if controller is within file, version will be 2
- if strings.Contains(line_controllers, cGroups[i].path) {
- *flag = true
- *flags_v2[j] = true
- }
-
- }
- j++
- if err := scanner.Err(); err != nil {
- logger.Debugf("Can't something")
- }
- }
+ for i, info := range cgroupsinfo {
+ if shared.PathExists("/sys/fs/cgroup/" + cGroups[i].path) {
+ // Check v1 support
+ *info = CGroupV1
+ } else if strings.Contains(v2controllers, cGroups[i].path) {
+ // Check v2 support
+ *info = CGroupV2
} else {
-
- *flag = shared.PathExists("/sys/fs/cgroup/" + cGroups[i].path)
- }
-
- if !*flag {
+ *info = CGroupDisabled
logger.Warnf(cGroups[i].warn)
}
}
diff --git a/lxd/sys/os.go b/lxd/sys/os.go
index c69565789d..b1c80a2ad3 100644
--- a/lxd/sys/os.go
+++ b/lxd/sys/os.go
@@ -30,6 +30,14 @@ type InotifyInfo struct {
Targets map[string]*InotifyTargetInfo
}
+// CGroupInfo records the cgroup support for a controller
+type CGroupInfo int
+const (
+ CGroupDisabled CGroupInfo = iota
+ CGroupV1
+ CGroupV2
+)
+
// OS is a high-level facade for accessing all operating-system
// level functionality that LXD uses.
type OS struct {
@@ -56,23 +64,19 @@ type OS struct {
AppArmorStacking bool
// Cgroup features
- CGroupBlkioController bool
- CGroupBlkioWeightController bool
- CGroupCPUacctController bool
- CGroupCPUController bool
- CGroupCPUsetController bool
- CGroupDevicesController bool
- CGroupFreezerController bool
- CGroupMemoryController bool
- CGroupNetPrioController bool
- CGroupPidsController bool
- CGroupSwapAccounting bool
-
- //detect if controller is v1 or v2
- CGroupMemoryControllerV2 bool
- CGroupCPUControllerV2 bool
- CGroupPidsControllerV2 bool
-
+ CGroupBlkioController CGroupInfo
+ CGroupBlkioWeightController CGroupInfo
+ CGroupCPUacctController CGroupInfo
+ CGroupCPUController CGroupInfo
+ CGroupCPUsetController CGroupInfo
+ CGroupDevicesController CGroupInfo
+ CGroupFreezerController CGroupInfo
+ CGroupMemoryController CGroupInfo
+ CGroupNetPrioController CGroupInfo
+ CGroupPidsController CGroupInfo
+ CGroupSwapAccounting CGroupInfo
+
+ // TODO not used
CGroupUnifiedHierarchy bool
From 8e330aa888ecdb8f9f36a941d6065d23c596faac Mon Sep 17 00:00:00 2001
From: Shreyas Sood <shreyassood at gmail.com>
Date: Mon, 9 Dec 2019 04:17:36 -0600
Subject: [PATCH 6/6] Memory Current Get
---
lxd/cgroup/cgroup.go | 15 ++++++++++++++-
lxd/container_lxc.go | 4 ++--
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/lxd/cgroup/cgroup.go b/lxd/cgroup/cgroup.go
index 756d9d6c11..9a5e8fa6b3 100644
--- a/lxd/cgroup/cgroup.go
+++ b/lxd/cgroup/cgroup.go
@@ -2,6 +2,7 @@ package cgroup
import (
"fmt"
+ "github.com/lxc/lxd/lxd/sys"
"gopkg.in/lxc/go-lxc.v2"
)
@@ -9,6 +10,7 @@ type Property int
const (
PidsCurrent Property = iota
PidsMax
+ MemoryCurrent
)
type configItem struct {
@@ -17,14 +19,25 @@ type configItem struct {
}
// Get finds property values on a lxcContainer
-func Get(c *lxc.Container, property Property) ([]string, error) {
+func Get(c *lxc.Container, os *sys.OS, property Property) ([]string, error) {
switch property {
+ // Current Memory Usage
+ case MemoryCurrent:
+ if os.CGroupMemoryController == sys.CGroupV2 {
+ return c.CgroupItem("memory.current"), nil
+ } else {
+ return c.CgroupItem("memory.usage_in_bytes"), nil
+ }
+
// Properties which have the same functionality for both v1 and v2
case PidsCurrent:
return c.CgroupItem("pids.current"), nil
+
}
+
+
return nil, fmt.Errorf("CGroup Property not supported for Get")
}
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index b7ed164b35..e85041d216 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -3783,7 +3783,7 @@ func (c *containerLXC) CGroupGet(property cgroup.Property) (string, error) {
return "", fmt.Errorf("Can't get cgroups on a stopped container")
}
- value, err := cgroup.Get(c.c, property)
+ value, err := cgroup.Get(c.c, c.state.OS, property)
if err != nil {
return "", err
@@ -5846,7 +5846,7 @@ func (c *containerLXC) memoryState() api.InstanceStateMemory {
}
// Memory in bytes
- value, err := c.CGroupGetV1("memory.usage_in_bytes")
+ value, err := c.CGroupGet(cgroup.MemoryCurrent)
valueInt, err1 := strconv.ParseInt(value, 10, 64)
if err == nil && err1 == nil {
memory.Usage = valueInt
More information about the lxc-devel
mailing list