[lxc-devel] [lxd/master] Isolated CPUs in resources API
stgraber on Github
lxc-bot at linuxcontainers.org
Wed Jun 17 16:42:18 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200617/0091b0b5/attachment.bin>
-------------- next part --------------
From 481a238a5c3b79530e89c71b044914a6eb41faeb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jun 2020 11:58:42 -0400
Subject: [PATCH 1/6] lxd/resources: Fix golint warning
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/resources/memory.go | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/lxd/resources/memory.go b/lxd/resources/memory.go
index 4a6544a659..b948f39e3b 100644
--- a/lxd/resources/memory.go
+++ b/lxd/resources/memory.go
@@ -153,14 +153,13 @@ func getTotalMemory() uint64 {
return 0
}
- var count uint64 = 0
-
entries, err := ioutil.ReadDir(sysDevicesSystemMemory)
if err != nil {
return 0
}
// Count the number of blocks
+ var count uint64
for _, entry := range entries {
// Only consider directories
if !entry.IsDir() {
From cafae2434c4f0b64463f7abb9494f561549f5328 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jun 2020 12:06:29 -0400
Subject: [PATCH 2/6] doc/api-extensions: Fix escaping
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
doc/api-extensions.md | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index ffe908dc68..4c3fbc1542 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -977,17 +977,17 @@ This adds a new `size` field to the output of `/1.0/instances/<name>/snapshots/<
This adds a writable endpoint for cluster members, allowing the editing of their roles.
## container\_nic\_routed\_host\_address
-This introduces the `ipv4.host_address` and `ipv6.host_address` NIC config keys that can be used to control the
+This introduces the `ipv4.host\_address` and `ipv6.host\_address` NIC config keys that can be used to control the
host-side veth interface's IP addresses. This can be useful when using multiple routed NICs at the same time and
needing a predictable next-hop address to use.
This also alters the behaviour of `ipv4.gateway` and `ipv6.gateway` NIC config keys. When they are set to "auto"
-the container will have its default gateway set to the value of `ipv4.host_address` or `ipv6.host_address` respectively.
+the container will have its default gateway set to the value of `ipv4.host\_address` or `ipv6.host\_address` respectively.
The default values are:
-`ipv4.host_address`: 169.254.0.1
-`ipv6.host_address`: fe80::1
+`ipv4.host\_address`: 169.254.0.1
+`ipv6.host\_address`: fe80::1
This is backward compatible with the previous default behaviour.
@@ -1015,11 +1015,11 @@ This introduces two new fields in `/1.0`, `os` and `os\_version`.
Those are taken from the os-release data on the system.
## container\_nic\_routed\_host\_table
-This introduces the `ipv4.host_table` and `ipv6.host_table` NIC config keys that can be used to add static routes
+This introduces the `ipv4.host\_table` and `ipv6.host\_table` NIC config keys that can be used to add static routes
for the instance's IPs to a custom policy routing table by ID.
## container\_nic\_ipvlan\_host\_table
-This introduces the `ipv4.host_table` and `ipv6.host_table` NIC config keys that can be used to add static routes
+This introduces the `ipv4.host\_table` and `ipv6.host\_table` NIC config keys that can be used to add static routes
for the instance's IPs to a custom policy routing table by ID.
## container\_nic\_ipvlan\_mode
From fb3583c9b612ede7dd7d49cae5b472c6b8ce3a0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jun 2020 12:08:18 -0400
Subject: [PATCH 3/6] api: resource_cpu_isolated
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
doc/api-extensions.md | 4 ++++
shared/api/resource.go | 3 +++
2 files changed, 7 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index 4c3fbc1542..a5d7c6a655 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -1073,3 +1073,7 @@ Bridge:
- Default VLAN
- VLAN filtering
- Upper devices
+
+## resources\_cpu\_isolated
+Add an `Isolated` property on CPU threads to indicate if the thread is
+physically `Online` but is configured not to accept tasks.
diff --git a/shared/api/resource.go b/shared/api/resource.go
index 2942972513..0a97eae196 100644
--- a/shared/api/resource.go
+++ b/shared/api/resource.go
@@ -74,6 +74,9 @@ type ResourcesCPUThread struct {
NUMANode uint64 `json:"numa_node" yaml:"numa_node"`
Thread uint64 `json:"thread" yaml:"thread"`
Online bool `json:"online" yaml:"online"`
+
+ // API extension: resource_cpu_isolated
+ Isolated bool `json:"isolated" yaml:"isolated"`
}
// ResourcesGPU represents the GPU resources available on the system
From 3f4fe3d2e4024843c996a2cb20277e71626d6e39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jun 2020 12:09:03 -0400
Subject: [PATCH 4/6] lxd/resources: Add Isolated property
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/resources/cpu.go | 66 ++++++++++++++++++++++++++++++++++++++++++
lxd/resources/utils.go | 9 ++++++
shared/api/resource.go | 2 +-
3 files changed, 76 insertions(+), 1 deletion(-)
diff --git a/lxd/resources/cpu.go b/lxd/resources/cpu.go
index 148bddfba7..70536f29f6 100644
--- a/lxd/resources/cpu.go
+++ b/lxd/resources/cpu.go
@@ -18,6 +18,68 @@ import (
var sysDevicesCPU = "/sys/devices/system/cpu"
+// GetCPUIsolated returns a slice of IDs corresponding to isolated threads.
+func GetCPUIsolated() []int64 {
+ isolatedPath := filepath.Join(sysDevicesCPU, "isolated")
+
+ isolatedCpusInt := []int64{}
+ if sysfsExists(isolatedPath) {
+ buf, err := ioutil.ReadFile(isolatedPath)
+ if err != nil {
+ return isolatedCpusInt
+ }
+
+ // File might exist even though there are no isolated cpus.
+ isolatedCpus := strings.TrimSpace(string(buf))
+ if isolatedCpus != "" {
+ isolatedCpusInt, err = ParseCpuset(isolatedCpus)
+ if err != nil {
+ return isolatedCpusInt
+ }
+ }
+ }
+
+ return isolatedCpusInt
+}
+
+// ParseCpuset parses a limits.cpu range into a list of CPU ids.
+func ParseCpuset(cpu string) ([]int64, error) {
+ cpus := []int64{}
+ chunks := strings.Split(cpu, ",")
+ for _, chunk := range chunks {
+ if strings.Contains(chunk, "-") {
+ // Range
+ fields := strings.SplitN(chunk, "-", 2)
+ if len(fields) != 2 {
+ return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
+ }
+
+ low, err := strconv.ParseInt(fields[0], 10, 64)
+ if err != nil {
+ return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
+ }
+
+ high, err := strconv.ParseInt(fields[1], 10, 64)
+ if err != nil {
+ return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
+ }
+
+ for i := low; i <= high; i++ {
+ cpus = append(cpus, i)
+ }
+ } else {
+ // Simple entry
+ nr, err := strconv.ParseInt(chunk, 10, 64)
+ if err != nil {
+ return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
+ }
+ cpus = append(cpus, nr)
+ }
+ }
+
+ return cpus, nil
+}
+
func getCPUCache(path string) ([]api.ResourcesCPUCache, error) {
caches := []api.ResourcesCPUCache{}
@@ -92,6 +154,9 @@ func getCPUCache(path string) ([]api.ResourcesCPUCache, error) {
func GetCPU() (*api.ResourcesCPU, error) {
cpu := api.ResourcesCPU{}
+ // Get the isolated CPUs
+ isolated := GetCPUIsolated()
+
// Temporary storage
cpuSockets := map[uint64]*api.ResourcesCPUSocket{}
cpuCores := map[uint64]map[string]*api.ResourcesCPUCore{}
@@ -299,6 +364,7 @@ func GetCPU() (*api.ResourcesCPU, error) {
}
}
thread.ID = threadNumber
+ thread.Isolated = int64InSlice(threadNumber, isolated)
thread.Thread = uint64(len(resCore.Threads))
// NUMA node
diff --git a/lxd/resources/utils.go b/lxd/resources/utils.go
index 8290b5c6ab..e5335e03e0 100644
--- a/lxd/resources/utils.go
+++ b/lxd/resources/utils.go
@@ -47,6 +47,15 @@ func stringInSlice(key string, list []string) bool {
return false
}
+func int64InSlice(key int64, list []int64) bool {
+ for _, entry := range list {
+ if entry == key {
+ return true
+ }
+ }
+ return false
+}
+
func sysfsExists(path string) bool {
_, err := os.Lstat(path)
if err == nil {
diff --git a/shared/api/resource.go b/shared/api/resource.go
index 0a97eae196..4ccefc43eb 100644
--- a/shared/api/resource.go
+++ b/shared/api/resource.go
@@ -76,7 +76,7 @@ type ResourcesCPUThread struct {
Online bool `json:"online" yaml:"online"`
// API extension: resource_cpu_isolated
- Isolated bool `json:"isolated" yaml:"isolated"`
+ Isolated bool `json:"isolated" yaml:"isolated"`
}
// ResourcesGPU represents the GPU resources available on the system
From b448225849304e84fa2514893b5f50d295a5389a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jun 2020 12:21:01 -0400
Subject: [PATCH 5/6] lxd/resources: Don't use shared
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/resources/system.go | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/lxd/resources/system.go b/lxd/resources/system.go
index bf0a7f494d..a074b54888 100644
--- a/lxd/resources/system.go
+++ b/lxd/resources/system.go
@@ -9,7 +9,6 @@ import (
"github.com/pkg/errors"
- "github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
)
@@ -134,14 +133,19 @@ func systemGetType() string {
return "unknown"
}
+ runDetectVirt := func(flag string) error {
+ cmd := exec.Command("systemd-detect-virt", flag)
+ return cmd.Run()
+ }
+
// If this returns 0, we're in a container.
- _, err = shared.RunCommand("systemd-detect-virt", "--container")
+ err = runDetectVirt("--container")
if err == nil {
return "container"
}
// If this returns 0, we're in a VM.
- _, err = shared.RunCommand("systemd-detect-virt", "--vm")
+ err = runDetectVirt("--vm")
if err == nil {
return "virtual-machine"
}
From 64acf92527cc0a04c31ea3a4d17e89f47bd71ea4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jun 2020 12:41:51 -0400
Subject: [PATCH 6/6] lxd/devices: Use resources for cpuset parsing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/devices.go | 38 ++++++++---------------------
lxd/instance/drivers/driver_qemu.go | 2 +-
lxd/instance/instance_utils.go | 37 ----------------------------
3 files changed, 11 insertions(+), 66 deletions(-)
diff --git a/lxd/devices.go b/lxd/devices.go
index 3442765652..65aa81ec44 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -2,7 +2,6 @@ package main
import (
"fmt"
- "io/ioutil"
"os"
"path"
"path/filepath"
@@ -16,6 +15,7 @@ import (
"github.com/lxc/lxd/lxd/device"
"github.com/lxc/lxd/lxd/instance"
"github.com/lxc/lxd/lxd/instance/instancetype"
+ "github.com/lxc/lxd/lxd/resources"
"github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/shared"
log "github.com/lxc/lxd/shared/log15"
@@ -55,7 +55,7 @@ static int get_hidraw_devinfo(int fd, struct hidraw_devinfo *info)
import "C"
type deviceTaskCPU struct {
- id int
+ id int64
strId string
count *int
}
@@ -337,34 +337,16 @@ func deviceTaskBalance(s *state.State) {
}
}
- effectiveCpusInt, err := instance.ParseCpuset(effectiveCpus)
+ effectiveCpusInt, err := resources.ParseCpuset(effectiveCpus)
if err != nil {
logger.Errorf("Error parsing effective CPU set")
return
}
- isolatedCpusInt := []int{}
- if shared.PathExists("/sys/devices/system/cpu/isolated") {
- buf, err := ioutil.ReadFile("/sys/devices/system/cpu/isolated")
- if err != nil {
- logger.Errorf("Error reading host's isolated cpu")
- return
- }
-
- // File might exist even though there are no isolated cpus.
- isolatedCpus := strings.TrimSpace(string(buf))
- if isolatedCpus != "" {
- isolatedCpusInt, err = instance.ParseCpuset(isolatedCpus)
- if err != nil {
- logger.Errorf("Error parsing isolated CPU set: %s", string(isolatedCpus))
- return
- }
- }
- }
-
+ isolatedCpusInt := resources.GetCPUIsolated()
effectiveCpusSlice := []string{}
for _, id := range effectiveCpusInt {
- if shared.IntInSlice(id, isolatedCpusInt) {
+ if shared.Int64InSlice(id, isolatedCpusInt) {
continue
}
@@ -377,7 +359,7 @@ func deviceTaskBalance(s *state.State) {
if err != nil && shared.PathExists("/sys/fs/cgroup/cpuset/lxc") {
logger.Warn("Error setting lxd's cpuset.cpus", log.Ctx{"err": err})
}
- cpus, err := instance.ParseCpuset(effectiveCpus)
+ cpus, err := resources.ParseCpuset(effectiveCpus)
if err != nil {
logger.Error("Error parsing host's cpu set", log.Ctx{"cpuset": effectiveCpus, "err": err})
return
@@ -390,7 +372,7 @@ func deviceTaskBalance(s *state.State) {
return
}
- fixedInstances := map[int][]instance.Instance{}
+ fixedInstances := map[int64][]instance.Instance{}
balancedInstances := map[instance.Instance]int{}
for _, c := range instances {
conf := c.ExpandedConfig()
@@ -410,12 +392,12 @@ func deviceTaskBalance(s *state.State) {
balancedInstances[c] = count
} else {
// Pinned
- containerCpus, err := instance.ParseCpuset(cpulimit)
+ containerCpus, err := resources.ParseCpuset(cpulimit)
if err != nil {
return
}
for _, nr := range containerCpus {
- if !shared.IntInSlice(nr, cpus) {
+ if !shared.Int64InSlice(nr, cpus) {
continue
}
@@ -431,7 +413,7 @@ func deviceTaskBalance(s *state.State) {
// Balance things
pinning := map[instance.Instance][]string{}
- usage := map[int]deviceTaskCPU{}
+ usage := map[int64]deviceTaskCPU{}
for _, id := range cpus {
cpu := deviceTaskCPU{}
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index dc7c9bbace..8316e05ab4 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -4458,7 +4458,7 @@ func (vm *qemu) cpuTopology(limit string) (int, int, int, map[uint64]uint64, map
}
// Expand the pins.
- pins, err := instance.ParseCpuset(limit)
+ pins, err := resources.ParseCpuset(limit)
if err != nil {
return -1, -1, -1, nil, nil, err
}
diff --git a/lxd/instance/instance_utils.go b/lxd/instance/instance_utils.go
index 9bbb85f4ee..e9837df99b 100644
--- a/lxd/instance/instance_utils.go
+++ b/lxd/instance/instance_utils.go
@@ -874,40 +874,3 @@ func ValidName(instanceName string, isSnapshot bool) error {
return nil
}
-
-// ParseCpuset parses a limits.cpu range into a list of CPU ids.
-func ParseCpuset(cpu string) ([]int, error) {
- cpus := []int{}
- chunks := strings.Split(cpu, ",")
- for _, chunk := range chunks {
- if strings.Contains(chunk, "-") {
- // Range
- fields := strings.SplitN(chunk, "-", 2)
- if len(fields) != 2 {
- return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
- }
-
- low, err := strconv.Atoi(fields[0])
- if err != nil {
- return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
- }
-
- high, err := strconv.Atoi(fields[1])
- if err != nil {
- return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
- }
-
- for i := low; i <= high; i++ {
- cpus = append(cpus, i)
- }
- } else {
- // Simple entry
- nr, err := strconv.Atoi(chunk)
- if err != nil {
- return nil, fmt.Errorf("Invalid cpuset value: %s", cpu)
- }
- cpus = append(cpus, nr)
- }
- }
- return cpus, nil
-}
More information about the lxc-devel
mailing list