[lxc-devel] [lxd/master] lxd/qemu: Use memory backend ram/file

[stgraber on Github]

A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 370 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200703/bb92b444/attachment.bin>
-------------- next part --------------
From 0a2d042d1ae876272df16dac5071887a99dd715d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Fri, 3 Jul 2020 17:17:55 -0400
Subject: [PATCH] lxd/qemu: Use memory backend ram/file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Closes #7567

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/instance/drivers/driver_qemu.go           | 82 ++++++++++---------
 lxd/instance/drivers/driver_qemu_templates.go | 20 ++++-
 2 files changed, 63 insertions(+), 39 deletions(-)

diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 0b2c5d13bc..11995dd1e0 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -777,16 +777,6 @@ func (vm *qemu) Start(stateful bool) error {
 		}
 	}
 
-	if shared.IsTrue(vm.expandedConfig["limits.memory.hugepages"]) {
-		hugetlb, err := util.HugepagesPath()
-		if err != nil {
-			op.Done(err)
-			return err
-		}
-
-		qemuCmd = append(qemuCmd, "-mem-path", hugetlb, "-mem-prealloc")
-	}
-
 	if vm.expandedConfig["raw.qemu"] != "" {
 		fields, err := shellquote.Split(vm.expandedConfig["raw.qemu"])
 		if err != nil {
@@ -1590,12 +1580,7 @@ func (vm *qemu) generateQemuConfigFile(busName string, devConfs []*deviceConfig.
 		return "", err
 	}
 
-	err = vm.addCPUConfig(sb)
-	if err != nil {
-		return "", err
-	}
-
-	err = vm.addMemoryConfig(sb)
+	err = vm.addCPUMemoryConfig(sb)
 	if err != nil {
 		return "", err
 	}
@@ -1798,27 +1783,8 @@ func (vm *qemu) generateQemuConfigFile(busName string, devConfs []*deviceConfig.
 	return configPath, ioutil.WriteFile(configPath, []byte(sb.String()), 0640)
 }
 
-// addMemoryConfig adds the qemu config required for setting the size of the VM's memory.
-func (vm *qemu) addMemoryConfig(sb *strings.Builder) error {
-	// Configure memory limit.
-	memSize := vm.expandedConfig["limits.memory"]
-	if memSize == "" {
-		memSize = "1GiB" // Default to 1GiB if no memory limit specified.
-	}
-
-	memSizeBytes, err := units.ParseByteSizeString(memSize)
-	if err != nil {
-		return fmt.Errorf("limits.memory invalid: %v", err)
-	}
-
-	return qemuMemory.Execute(sb, map[string]interface{}{
-		"architecture": vm.architectureName,
-		"memSizeBytes": memSizeBytes,
-	})
-}
-
-// addCPUConfig adds the qemu config required for setting the number of virtualised CPUs.
-func (vm *qemu) addCPUConfig(sb *strings.Builder) error {
+// addCPUMemoryConfig adds the qemu config required for setting the number of virtualised CPUs and memory.
+func (vm *qemu) addCPUMemoryConfig(sb *strings.Builder) error {
 	// Default to a single core.
 	cpus := vm.expandedConfig["limits.cpu"]
 	if cpus == "" {
@@ -1830,6 +1796,7 @@ func (vm *qemu) addCPUConfig(sb *strings.Builder) error {
 	}
 
 	cpuCount, err := strconv.Atoi(cpus)
+	hostNodes := []uint64{}
 	if err == nil {
 		// If not pinning, default to exposing cores.
 		ctx["cpuCount"] = cpuCount
@@ -1863,7 +1830,9 @@ func (vm *qemu) addCPUConfig(sb *strings.Builder) error {
 		numa := []map[string]uint64{}
 		numaIDs := []uint64{}
 		numaNode := uint64(0)
-		for _, entry := range numaNodes {
+		for hostNode, entry := range numaNodes {
+			hostNodes = append(hostNodes, hostNode)
+
 			numaIDs = append(numaIDs, numaNode)
 			for _, vcpu := range entry {
 				numa = append(numa, map[string]uint64{
@@ -1884,8 +1853,45 @@ func (vm *qemu) addCPUConfig(sb *strings.Builder) error {
 		ctx["cpuThreads"] = nrThreads
 		ctx["cpuNumaNodes"] = numaIDs
 		ctx["cpuNumaMapping"] = numa
+		ctx["cpuNumaHostNodes"] = hostNodes
+	}
+
+	// Configure memory limit.
+	memSize := vm.expandedConfig["limits.memory"]
+	if memSize == "" {
+		memSize = "1GiB" // Default to 1GiB if no memory limit specified.
+	}
+
+	memSizeBytes, err := units.ParseByteSizeString(memSize)
+	if err != nil {
+		return fmt.Errorf("limits.memory invalid: %v", err)
+	}
+
+	ctx["hugepages"] = ""
+	if shared.IsTrue(vm.expandedConfig["limits.memory.hugepages"]) {
+		hugetlb, err := util.HugepagesPath()
+		if err != nil {
+			return err
+		}
+
+		ctx["hugepages"] = hugetlb
+	}
+
+	// Determine per-node memory limit.
+	memSizeBytes = memSizeBytes / 1024 / 1024
+	nodeMemory := int64(memSizeBytes / int64(len(hostNodes)))
+	memSizeBytes = nodeMemory * int64(len(hostNodes))
+	ctx["memory"] = nodeMemory
+
+	err = qemuMemory.Execute(sb, map[string]interface{}{
+		"architecture": vm.architectureName,
+		"memSizeBytes": memSizeBytes,
+	})
+	if err != nil {
+		return err
 	}
 
+	// Configure the CPU limit.
 	return qemuCPU.Execute(sb, ctx)
 }
 
diff --git a/lxd/instance/drivers/driver_qemu_templates.go b/lxd/instance/drivers/driver_qemu_templates.go
index ee7943ddd9..41006e9892 100644
--- a/lxd/instance/drivers/driver_qemu_templates.go
+++ b/lxd/instance/drivers/driver_qemu_templates.go
@@ -55,7 +55,7 @@ disable-ticketing = "on"
 var qemuMemory = template.Must(template.New("qemuMemory").Parse(`
 # Memory
 [memory]
-size = "{{.memSizeBytes}}B"
+size = "{{.memSizeBytes}}M"
 `))
 
 var qemuSerial = template.Must(template.New("qemuSerial").Parse(`
@@ -226,10 +226,28 @@ cores = "{{.cpuCores}}"
 threads = "{{.cpuThreads}}"
 
 {{if eq .architecture "x86_64" -}}
+{{$memory := .memory -}}
+{{$hugepages := .hugepages -}}
+{{range $index, $element := .cpuNumaHostNodes}}
+[object "mem{{$index}}"]
+{{if ne $hugepages "" -}}
+qom-type = "memory-backend-file"
+mem-path = "{{$hugepages}}"
+prealloc = "on"
+discard-data = "on"
+{{- else}}
+qom-type = "memory-backend-ram"
+{{- end }}
+size = "{{$memory}}M"
+host-nodes = "{{$element}}"
+policy = "bind"
+{{end}}
+
 {{range $index, $element := .cpuNumaNodes}}
 [numa]
 type = "node"
 nodeid = "{{$element}}"
+memdev = "mem{{$element}}"
 {{end}}
 
 {{range .cpuNumaMapping}}