[lxc-devel] [lxd/master] VM: PCIe addressing take two
tomponline on Github
lxc-bot at linuxcontainers.org
Fri Apr 17 10:32:36 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 743 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200417/ec86a9e5/attachment.bin>
-------------- next part --------------
From 143ebba26f2ab6b1a2752451d5b681022fc5576e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 17 Apr 2020 11:28:23 +0100
Subject: [PATCH 1/2] lxd/instance/drivers/driver/qemu/templates: Use static
PCIe address prefix for 9p devices
Uses 0x3.0x as address prefix to avoid qemu auto generating the address and causing conflicts when ordering means they are added in between NICs being added.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/instance/drivers/driver_qemu_templates.go | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/lxd/instance/drivers/driver_qemu_templates.go b/lxd/instance/drivers/driver_qemu_templates.go
index ad65921e09..9eb9079387 100644
--- a/lxd/instance/drivers/driver_qemu_templates.go
+++ b/lxd/instance/drivers/driver_qemu_templates.go
@@ -187,6 +187,7 @@ unit = "1"
`))
// Devices use "qemu_" prefix indicating that this is a internally named device.
+// Use 0x3.0x as the PCIe address prefix for 9p disk devices to allow up to 8 devices of this type.
var qemuDriveConfig = template.Must(template.New("qemuDriveConfig").Parse(`
# Config drive
[fsdev "qemu_config"]
@@ -199,9 +200,12 @@ path = "{{.path}}"
driver = "virtio-9p-pci"
fsdev = "qemu_config"
mount_tag = "config"
+multifunction = "on"
+addr = "0x3.0x{{.diskIndex}}"
`))
// Devices use "lxd_" prefix indicating that this is a user named device.
+// Use 0x3.0x as the PCIe address prefix for 9p disk devices to allow up to 8 devices of this type.
var qemuDriveDir = template.Must(template.New("qemuDriveDir").Parse(`
# {{.devName}} drive
[fsdev "lxd_{{.devName}}"]
@@ -220,6 +224,8 @@ sock_fd = "{{.proxyFD}}"
driver = "virtio-9p-pci"
fsdev = "lxd_{{.devName}}"
mount_tag = "{{.mountTag}}"
+multifunction = "on"
+addr = "0x3.0x{{.diskIndex}}"
`))
// Devices use "lxd_" prefix indicating that this is a user named device.
@@ -246,6 +252,7 @@ bootindex = "{{.bootIndex}}"
`))
// qemuDevTapCommon is common PCI device template for tap based netdevs.
+// Use 0x4.0x as the PCIe address prefix for nic devices to allow up to 8 devices of this type.
var qemuDevTapCommon = template.Must(template.New("qemuDevTapCommon").Parse(`
{{if ne .architecture "ppc64le" -}}
[device "qemu_pcie{{.chassisIndex}}"]
From edc887f40a818bdabd1cbf78a6cc192791efd4a3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 17 Apr 2020 11:29:30 +0100
Subject: [PATCH 2/2] lxd/instance/drivers/drivers/qemu: Adds support for 9p
disk device PCIe indexes
Avoids ordering issues.
Improves comments around PCIe indexing.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/instance/drivers/driver_qemu.go | 46 +++++++++++++++++++----------
1 file changed, 30 insertions(+), 16 deletions(-)
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 0b83a30380..38df3d9e5c 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -1562,12 +1562,19 @@ func (vm *qemu) generateQemuConfigFile(devConfs []*deviceConfig.RunConfig, fdFil
return "", err
}
- err = vm.addConfDriveConfig(sb)
+ // Indexes used for PCIe address generation (each device type group is assigned their own PCIe address
+ // prefix in the templates). Each PCIe device is added as a multifunction device allowing up to 8 devices
+ // of each type to be added.
+ nicIndex := 0
+ diskIndex := 0
+ chassisIndex := 5 // Internal devices defined in the templates use indexes 1-4.
+
+ err = vm.addConfDriveConfig(sb, diskIndex)
if err != nil {
return "", err
}
+ diskIndex++ // The config drive is a 9p device which uses a PCIe function so increment index.
- nicIndex := 0
bootIndexes, err := vm.deviceBootPriorities()
if err != nil {
return "", errors.Wrap(err, "Error calculating boot indexes")
@@ -1583,7 +1590,8 @@ func (vm *qemu) generateQemuConfigFile(devConfs []*deviceConfig.RunConfig, fdFil
if drive.TargetPath == "/" {
err = vm.addRootDriveConfig(sb, bootIndexes, drive)
} else if drive.FSType == "9p" {
- err = vm.addDriveDirConfig(sb, fdFiles, &agentMounts, drive)
+ err = vm.addDriveDirConfig(sb, diskIndex, fdFiles, &agentMounts, drive)
+ diskIndex++ // 9p devices use a PCIe function so increment index.
} else {
err = vm.addDriveConfig(sb, bootIndexes, drive)
}
@@ -1595,11 +1603,14 @@ func (vm *qemu) generateQemuConfigFile(devConfs []*deviceConfig.RunConfig, fdFil
// Add network device.
if len(runConf.NetworkInterface) > 0 {
- err = vm.addNetDevConfig(sb, nicIndex, bootIndexes, runConf.NetworkInterface, fdFiles)
+ err = vm.addNetDevConfig(sb, chassisIndex, nicIndex, bootIndexes, runConf.NetworkInterface, fdFiles)
if err != nil {
return "", err
}
+
+ // NIC devices use a PCIe function so increment indexes.
nicIndex++
+ chassisIndex++
}
}
@@ -1705,10 +1716,11 @@ func (vm *qemu) addFirmwareConfig(sb *strings.Builder) error {
}
// addConfDriveConfig adds the qemu config required for adding the config drive.
-func (vm *qemu) addConfDriveConfig(sb *strings.Builder) error {
+func (vm *qemu) addConfDriveConfig(sb *strings.Builder, diskIndex int) error {
return qemuDriveConfig.Execute(sb, map[string]interface{}{
"architecture": vm.architectureName,
"path": filepath.Join(vm.Path(), "config"),
+ "diskIndex": diskIndex,
})
}
@@ -1754,7 +1766,7 @@ func (vm *qemu) addRootDriveConfig(sb *strings.Builder, bootIndexes map[string]i
}
// addDriveDirConfig adds the qemu config required for adding a supplementary drive directory share.
-func (vm *qemu) addDriveDirConfig(sb *strings.Builder, fdFiles *[]string, agentMounts *[]instancetype.VMAgentMount, driveConf deviceConfig.MountEntryItem) error {
+func (vm *qemu) addDriveDirConfig(sb *strings.Builder, diskIndex int, fdFiles *[]string, agentMounts *[]instancetype.VMAgentMount, driveConf deviceConfig.MountEntryItem) error {
mountTag := fmt.Sprintf("lxd_%s", driveConf.DevName)
agentMount := instancetype.VMAgentMount{
@@ -1775,20 +1787,22 @@ func (vm *qemu) addDriveDirConfig(sb *strings.Builder, fdFiles *[]string, agentM
// For read only shares, do not use proxy.
if shared.StringInSlice("ro", driveConf.Opts) {
return qemuDriveDir.Execute(sb, map[string]interface{}{
- "devName": driveConf.DevName,
- "mountTag": mountTag,
- "path": driveConf.DevPath,
- "readonly": true,
+ "devName": driveConf.DevName,
+ "mountTag": mountTag,
+ "path": driveConf.DevPath,
+ "readonly": true,
+ "diskIndex": diskIndex,
})
}
// Only use proxy for writable shares.
proxyFD := vm.addFileDescriptor(fdFiles, driveConf.DevPath)
return qemuDriveDir.Execute(sb, map[string]interface{}{
- "devName": driveConf.DevName,
- "mountTag": mountTag,
- "proxyFD": proxyFD,
- "readonly": false,
+ "devName": driveConf.DevName,
+ "mountTag": mountTag,
+ "proxyFD": proxyFD,
+ "readonly": false,
+ "diskIndex": diskIndex,
})
}
@@ -1830,7 +1844,7 @@ func (vm *qemu) addDriveConfig(sb *strings.Builder, bootIndexes map[string]int,
}
// addNetDevConfig adds the qemu config required for adding a network device.
-func (vm *qemu) addNetDevConfig(sb *strings.Builder, nicIndex int, bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem, fdFiles *[]string) error {
+func (vm *qemu) addNetDevConfig(sb *strings.Builder, chassisIndex, nicIndex int, bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem, fdFiles *[]string) error {
var devName, nicName, devHwaddr, pciSlotName string
for _, nicItem := range nicConfig {
if nicItem.Key == "devName" {
@@ -1851,7 +1865,7 @@ func (vm *qemu) addNetDevConfig(sb *strings.Builder, nicIndex int, bootIndexes m
"devHwaddr": devHwaddr,
"bootIndex": bootIndexes[devName],
"nicIndex": nicIndex,
- "chassisIndex": 5 + nicIndex,
+ "chassisIndex": chassisIndex,
}
// Detect MACVTAP interface types and figure out which tap device is being used.
More information about the lxc-devel
mailing list