[lxc-devel] [lxd/master] CPU pinning for virtual machines
stgraber on Github
lxc-bot at linuxcontainers.org
Tue Feb 11 23:56:44 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200211/774e2af3/attachment.bin>
-------------- next part --------------
From 641a9f9bfc73c8ffe42154f6b364cc5ffd8597b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 11 Feb 2020 15:32:17 -0800
Subject: [PATCH 1/3] lxd/instance: Move ParseCpuset
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/devices.go | 44 ++++------------------------------
lxd/instance/instance_utils.go | 37 ++++++++++++++++++++++++++++
2 files changed, 41 insertions(+), 40 deletions(-)
diff --git a/lxd/devices.go b/lxd/devices.go
index 497960f2af..4ce79fa9ae 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -285,42 +285,6 @@ func deviceNetlinkListener() (chan []string, chan []string, chan device.USBEvent
return chCPU, chNetwork, chUSB, chUnix, nil
}
-func parseCpuset(cpu string) ([]int, error) {
- cpus := []int{}
- chunks := strings.Split(cpu, ",")
- for _, chunk := range chunks {
- if strings.Contains(chunk, "-") {
- // Range
- fields := strings.SplitN(chunk, "-", 2)
- if len(fields) != 2 {
- return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
- }
-
- low, err := strconv.Atoi(fields[0])
- if err != nil {
- return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
- }
-
- high, err := strconv.Atoi(fields[1])
- if err != nil {
- return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
- }
-
- for i := low; i <= high; i++ {
- cpus = append(cpus, i)
- }
- } else {
- // Simple entry
- nr, err := strconv.Atoi(chunk)
- if err != nil {
- return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
- }
- cpus = append(cpus, nr)
- }
- }
- return cpus, nil
-}
-
func deviceTaskBalance(s *state.State) {
min := func(x, y int) int {
if x < y {
@@ -345,7 +309,7 @@ func deviceTaskBalance(s *state.State) {
}
}
- effectiveCpusInt, err := parseCpuset(effectiveCpus)
+ effectiveCpusInt, err := instance.ParseCpuset(effectiveCpus)
if err != nil {
logger.Errorf("Error parsing effective CPU set")
return
@@ -362,7 +326,7 @@ func deviceTaskBalance(s *state.State) {
// File might exist even though there are no isolated cpus.
isolatedCpus := strings.TrimSpace(string(buf))
if isolatedCpus != "" {
- isolatedCpusInt, err = parseCpuset(isolatedCpus)
+ isolatedCpusInt, err = instance.ParseCpuset(isolatedCpus)
if err != nil {
logger.Errorf("Error parsing isolated CPU set: %s", string(isolatedCpus))
return
@@ -385,7 +349,7 @@ func deviceTaskBalance(s *state.State) {
if err != nil && shared.PathExists("/sys/fs/cgroup/cpuset/lxc") {
logger.Warn("Error setting lxd's cpuset.cpus", log.Ctx{"err": err})
}
- cpus, err := parseCpuset(effectiveCpus)
+ cpus, err := instance.ParseCpuset(effectiveCpus)
if err != nil {
logger.Error("Error parsing host's cpu set", log.Ctx{"cpuset": effectiveCpus, "err": err})
return
@@ -418,7 +382,7 @@ func deviceTaskBalance(s *state.State) {
balancedInstances[c] = count
} else {
// Pinned
- containerCpus, err := parseCpuset(cpulimit)
+ containerCpus, err := instance.ParseCpuset(cpulimit)
if err != nil {
return
}
diff --git a/lxd/instance/instance_utils.go b/lxd/instance/instance_utils.go
index 7dcc748da9..ae707c1258 100644
--- a/lxd/instance/instance_utils.go
+++ b/lxd/instance/instance_utils.go
@@ -958,3 +958,40 @@ func ValidName(instanceName string, isSnapshot bool) error {
return nil
}
+
+// ParseCpuset parses a limits.cpu range into a list of CPU ids.
+func ParseCpuset(cpu string) ([]int, error) {
+ cpus := []int{}
+ chunks := strings.Split(cpu, ",")
+ for _, chunk := range chunks {
+ if strings.Contains(chunk, "-") {
+ // Range
+ fields := strings.SplitN(chunk, "-", 2)
+ if len(fields) != 2 {
+ return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
+ }
+
+ low, err := strconv.Atoi(fields[0])
+ if err != nil {
+ return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
+ }
+
+ high, err := strconv.Atoi(fields[1])
+ if err != nil {
+ return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
+ }
+
+ for i := low; i <= high; i++ {
+ cpus = append(cpus, i)
+ }
+ } else {
+ // Simple entry
+ nr, err := strconv.Atoi(chunk)
+ if err != nil {
+ return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
+ }
+ cpus = append(cpus, nr)
+ }
+ }
+ return cpus, nil
+}
From 744a23bb6eeaee7850bf701d938c70a0772fe11f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 11 Feb 2020 15:53:24 -0800
Subject: [PATCH 2/3] lxd/vm/qmp: Allow retrieving vCPU pids
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/instance/drivers/qmp/monitor.go | 36 +++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/lxd/instance/drivers/qmp/monitor.go b/lxd/instance/drivers/qmp/monitor.go
index 4b96312228..1d81f5b8ab 100644
--- a/lxd/instance/drivers/qmp/monitor.go
+++ b/lxd/instance/drivers/qmp/monitor.go
@@ -284,3 +284,39 @@ func (m *Monitor) Quit() error {
func (m *Monitor) AgentReady() bool {
return m.agentReady
}
+
+// GetCPUs fetches the vCPU information for pinning.
+func (m *Monitor) GetCPUs() ([]int, error) {
+ // Check if disconnected
+ if m.disconnected {
+ return nil, ErrMonitorDisconnect
+ }
+
+ // Query the consoles.
+ respRaw, err := m.qmp.Run([]byte("{'execute': 'query-cpus'}"))
+ if err != nil {
+ m.Disconnect()
+ return nil, ErrMonitorDisconnect
+ }
+
+ // Process the response.
+ var respDecoded struct {
+ Return []struct {
+ CPU int `json:"CPU"`
+ PID int `json:"thread_id"`
+ } `json:"return"`
+ }
+
+ err = json.Unmarshal(respRaw, &respDecoded)
+ if err != nil {
+ return nil, ErrMonitorBadReturn
+ }
+
+ // Make a slice of PIDs.
+ pids := []int{}
+ for _, cpu := range respDecoded.Return {
+ pids = append(pids, cpu.PID)
+ }
+
+ return pids, nil
+}
From 1d0ad4eb79d594c305eb7b9ccb4669ebd3f3bc3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 11 Feb 2020 15:53:40 -0800
Subject: [PATCH 3/3] lxd/vm: Implement CPU pinning
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/instance/drivers/driver_qemu.go | 47 ++++++++++++++++++++++++++++-
1 file changed, 46 insertions(+), 1 deletion(-)
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index efb2276fbb..28c6f3256d 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -819,6 +819,46 @@ func (vm *qemu) Start(stateful bool) error {
return err
}
+ // Apply CPU pinning.
+ cpuLimit, ok := vm.expandedConfig["limits.cpu"]
+ if ok && cpuLimit != "" {
+ _, err := strconv.Atoi(cpuLimit)
+ if err != nil {
+ // Expand to a set of CPU identifiers.
+ pins, err := instance.ParseCpuset(cpuLimit)
+ if err != nil {
+ op.Done(err)
+ return err
+ }
+
+ // Get the list of PIDs from the VM.
+ pids, err := monitor.GetCPUs()
+ if err != nil {
+ op.Done(err)
+ return err
+ }
+
+ // Confirm nothing weird is going on.
+ if len(pins) != len(pids) {
+ return fmt.Errorf("QEMU has less vCPUs than configured")
+ }
+
+ for i, pin := range pins {
+ pid := pids[i]
+
+ set := unix.CPUSet{}
+ set.Set(pin)
+
+ // Apply the pin.
+ err := unix.SchedSetaffinity(pid, &set)
+ if err != nil {
+ op.Done(err)
+ return err
+ }
+ }
+ }
+ }
+
// Start the VM.
err = monitor.Start()
if err != nil {
@@ -1386,7 +1426,12 @@ func (vm *qemu) addCPUConfig(sb *strings.Builder) error {
cpuCount, err := strconv.Atoi(cpus)
if err != nil {
- return fmt.Errorf("limits.cpu invalid: %v", err)
+ pins, err := instance.ParseCpuset(cpus)
+ if err != nil {
+ return fmt.Errorf("limits.cpu invalid: %v", err)
+ }
+
+ cpuCount = len(pins)
}
return qemuCPU.Execute(sb, map[string]interface{}{
More information about the lxc-devel
mailing list