[lxc-devel] [lxd/master] lxd/instance/qemu: Enable multiqueue on tap NICs

stgraber on Github lxc-bot at linuxcontainers.org
Thu Dec 17 17:28:24 UTC 2020


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 354 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20201217/87de2be5/attachment.bin>
-------------- next part --------------
From dac33675626fb7b04ff6d2af9b0809ee776cbea7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 17 Dec 2020 12:28:08 -0500
Subject: [PATCH] lxd/instance/qemu: Enable multiqueue on tap NICs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/device/device_utils_network.go            |  2 +-
 lxd/instance/drivers/driver_qemu.go           | 22 +++++++++++--------
 lxd/instance/drivers/driver_qemu_templates.go |  7 ++++++
 3 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/lxd/device/device_utils_network.go b/lxd/device/device_utils_network.go
index 74483dbce1..986222a124 100644
--- a/lxd/device/device_utils_network.go
+++ b/lxd/device/device_utils_network.go
@@ -260,7 +260,7 @@ func networkCreateVethPair(hostName string, m deviceConfig.Device) (string, erro
 
 // networkCreateTap creates and configures a TAP device.
 func networkCreateTap(hostName string, m deviceConfig.Device) error {
-	_, err := shared.RunCommand("ip", "tuntap", "add", "name", hostName, "mode", "tap")
+	_, err := shared.RunCommand("ip", "tuntap", "add", "name", hostName, "mode", "tap", "multi_queue")
 	if err != nil {
 		return errors.Wrapf(err, "Failed to create the tap interfaces %s", hostName)
 	}
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 287073a2f1..4e0dd2eb2a 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -1805,7 +1805,7 @@ func (d *qemu) generateQemuConfigFile(mountInfo *storagePools.MountInfo, busName
 		return "", err
 	}
 
-	err = d.addCPUMemoryConfig(sb)
+	cpuCount, err := d.addCPUMemoryConfig(sb)
 	if err != nil {
 		return "", err
 	}
@@ -2008,7 +2008,7 @@ func (d *qemu) generateQemuConfigFile(mountInfo *storagePools.MountInfo, busName
 
 		// Add network device.
 		if len(runConf.NetworkInterface) > 0 {
-			err = d.addNetDevConfig(sb, bus, bootIndexes, runConf.NetworkInterface, fdFiles)
+			err = d.addNetDevConfig(sb, cpuCount, bus, bootIndexes, runConf.NetworkInterface, fdFiles)
 			if err != nil {
 				return "", err
 			}
@@ -2058,7 +2058,7 @@ func (d *qemu) generateQemuConfigFile(mountInfo *storagePools.MountInfo, busName
 }
 
 // addCPUMemoryConfig adds the qemu config required for setting the number of virtualised CPUs and memory.
-func (d *qemu) addCPUMemoryConfig(sb *strings.Builder) error {
+func (d *qemu) addCPUMemoryConfig(sb *strings.Builder) (int, error) {
 	// Default to a single core.
 	cpus := d.expandedConfig["limits.cpu"]
 	if cpus == "" {
@@ -2082,7 +2082,7 @@ func (d *qemu) addCPUMemoryConfig(sb *strings.Builder) error {
 		// Expand to a set of CPU identifiers and get the pinning map.
 		nrSockets, nrCores, nrThreads, vcpus, numaNodes, err := d.cpuTopology(cpus)
 		if err != nil {
-			return err
+			return -1, err
 		}
 
 		// Figure out socket-id/core-id/thread-id for all vcpus.
@@ -2139,14 +2139,14 @@ func (d *qemu) addCPUMemoryConfig(sb *strings.Builder) error {
 
 	memSizeBytes, err := units.ParseByteSizeString(memSize)
 	if err != nil {
-		return fmt.Errorf("limits.memory invalid: %v", err)
+		return -1, fmt.Errorf("limits.memory invalid: %v", err)
 	}
 
 	ctx["hugepages"] = ""
 	if shared.IsTrue(d.expandedConfig["limits.memory.hugepages"]) {
 		hugetlb, err := util.HugepagesPath()
 		if err != nil {
-			return err
+			return -1, err
 		}
 
 		ctx["hugepages"] = hugetlb
@@ -2163,11 +2163,11 @@ func (d *qemu) addCPUMemoryConfig(sb *strings.Builder) error {
 		"memSizeBytes": memSizeBytes,
 	})
 	if err != nil {
-		return err
+		return -1, err
 	}
 
 	// Configure the CPU limit.
-	return qemuCPU.Execute(sb, ctx)
+	return ctx["cpuCount"].(int), qemuCPU.Execute(sb, ctx)
 }
 
 // addFileDescriptor adds a file path to the list of files to open and pass file descriptor to qemu.
@@ -2336,7 +2336,7 @@ func (d *qemu) addDriveConfig(sb *strings.Builder, bootIndexes map[string]int, d
 }
 
 // addNetDevConfig adds the qemu config required for adding a network device.
-func (d *qemu) addNetDevConfig(sb *strings.Builder, bus *qemuBus, bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem, fdFiles *[]string) error {
+func (d *qemu) addNetDevConfig(sb *strings.Builder, cpuCount int, bus *qemuBus, bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem, fdFiles *[]string) error {
 	var devName, nicName, devHwaddr, pciSlotName string
 	for _, nicItem := range nicConfig {
 		if nicItem.Key == "devName" {
@@ -2355,6 +2355,8 @@ func (d *qemu) addNetDevConfig(sb *strings.Builder, bus *qemuBus, bootIndexes ma
 		"bus":       bus.name,
 		"devName":   devName,
 		"devHwaddr": devHwaddr,
+		"vectors":   0,
+		"queues":    0,
 		"bootIndex": bootIndexes[devName],
 	}
 
@@ -2377,6 +2379,8 @@ func (d *qemu) addNetDevConfig(sb *strings.Builder, bus *qemuBus, bootIndexes ma
 	} else if shared.PathExists(fmt.Sprintf("/sys/class/net/%s/tun_flags", nicName)) {
 		// Detect TAP (via TUN driver) device.
 		tplFields["ifName"] = nicName
+		tplFields["vectors"] = 2*cpuCount + 2
+		tplFields["queues"] = cpuCount
 		tpl = qemuNetDevTapTun
 	} else if pciSlotName != "" {
 		// Detect physical passthrough device.
diff --git a/lxd/instance/drivers/driver_qemu_templates.go b/lxd/instance/drivers/driver_qemu_templates.go
index 68c1927f67..66705f74af 100644
--- a/lxd/instance/drivers/driver_qemu_templates.go
+++ b/lxd/instance/drivers/driver_qemu_templates.go
@@ -469,6 +469,10 @@ driver = "virtio-net-ccw"
 {{- end}}
 netdev = "lxd_{{.devName}}"
 mac = "{{.devHwaddr}}"
+{{ if ne .vectors 0 -}}
+mq = "on"
+vectors = "{{.vectors}}"
+{{- end}}
 bootindex = "{{.bootIndex}}"
 {{if .multifunction -}}
 multifunction = "on"
@@ -481,6 +485,9 @@ var qemuNetDevTapTun = template.Must(qemuNetDevTapCommon.New("qemuNetDevTapTun")
 [netdev "lxd_{{.devName}}"]
 type = "tap"
 vhost = "on"
+{{ if ne .queues 0 -}}
+queues = "{{.queues}}"
+{{- end}}
 ifname = "{{.ifName}}"
 script = "no"
 downscript = "no"


More information about the lxc-devel mailing list