[lxc-devel] [lxd/master] lxd/vm: Move bus allocator to own file
stgraber on Github
lxc-bot at linuxcontainers.org
Thu Jun 11 23:47:06 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 354 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200611/2f43f1ec/attachment.bin>
-------------- next part --------------
From 69598929e8eac6447019dd5087ec38534fec72b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 11 Jun 2020 19:46:38 -0400
Subject: [PATCH] lxd/vm: Move bus allocator to own file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/instance/drivers/driver_qemu.go | 170 +++++-------------------
lxd/instance/drivers/driver_qemu_bus.go | 142 ++++++++++++++++++++
2 files changed, 172 insertions(+), 140 deletions(-)
create mode 100644 lxd/instance/drivers/driver_qemu_bus.go
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index c2a81f479f..7c12346a98 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -1548,7 +1548,7 @@ func (vm *qemu) deviceBootPriorities() (map[string]int, error) {
// generateQemuConfigFile writes the qemu config file and returns its location.
// It writes the config file inside the VM's log path.
-func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunConfig, fdFiles *[]string) (string, error) {
+func (vm *qemu) generateQemuConfigFile(busName string, devConfs []*deviceConfig.RunConfig, fdFiles *[]string) (string, error) {
var sb *strings.Builder = &strings.Builder{}
err := qemuBase.Execute(sb, map[string]interface{}{
@@ -1585,123 +1585,13 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
return "", err
}
- // allocateBusAddress is used to create any needed root ports and provide
- // the bus and address that should be used by a device. It supports
- // automatically setting up multi-function devices and optimize their use.
- portNum := 0
- devNum := 1
-
- type entry struct {
- bridgeDev int // Device number on the root bridge.
- bridgeFn int // Function number on the root bridge.
-
- dev string // Existing device name.
- fn int // Function number on the existing device.
- }
- entries := map[string]*entry{}
-
- var rootPort *entry
- allocateRootPort := func() *entry {
- if rootPort == nil {
- rootPort = &entry{
- bridgeDev: devNum,
- }
- devNum++
- } else {
- if rootPort.bridgeFn == 7 {
- rootPort.bridgeFn = 0
- rootPort.bridgeDev = devNum
- devNum++
- } else {
- rootPort.bridgeFn++
- }
- }
-
- return rootPort
- }
-
- allocateBusAddr := func(group string) (string, string, bool) {
- // FIXME: Need to figure out if ccw needs any bus logic.
- if bus == "ccw" {
- return "", "", false
- }
-
- // Find a device group if specified.
- var p *entry
- if group != "" {
- var ok bool
- p, ok = entries[group]
- if ok {
- // Check if group is full.
- if p.fn == 7 {
- p.fn = 0
- if bus == "pci" {
- p.bridgeDev = devNum
- devNum++
- } else if bus == "pcie" {
- r := allocateRootPort()
- p.bridgeDev = r.bridgeDev
- p.bridgeFn = r.bridgeFn
- }
- } else {
- p.fn++
- }
- } else {
- // Create a new group.
- p = &entry{}
-
- if bus == "pci" {
- p.bridgeDev = devNum
- devNum++
- } else if bus == "pcie" {
- r := allocateRootPort()
- p.bridgeDev = r.bridgeDev
- p.bridgeFn = r.bridgeFn
- }
-
- entries[group] = p
- }
- } else {
- // Create a new temporary group.
- p = &entry{}
-
- if bus == "pci" {
- p.bridgeDev = devNum
- devNum++
- } else if bus == "pcie" {
- r := allocateRootPort()
- p.bridgeDev = r.bridgeDev
- p.bridgeFn = r.bridgeFn
- }
- }
-
- multi := p.fn == 0 && group != ""
-
- if bus == "pci" {
- return "pci.0", fmt.Sprintf("%x.%d", p.bridgeDev, p.fn), multi
- }
-
- if bus == "pcie" {
- if p.fn == 0 {
- qemuPCIe.Execute(sb, map[string]interface{}{
- "index": portNum,
- "addr": fmt.Sprintf("%x.%d", p.bridgeDev, p.bridgeFn),
- "multifunction": p.bridgeFn == 0,
- })
- p.dev = fmt.Sprintf("qemu_pcie%d", portNum)
- portNum++
- }
-
- return p.dev, fmt.Sprintf("00.%d", p.fn), multi
- }
-
- return "", "", false
- }
+ // Setup the bus allocator.
+ bus := qemuNewBus(busName, sb)
// Now add the fixed set of devices.
- devBus, devAddr, multi := allocateBusAddr("generic")
+ devBus, devAddr, multi := bus.allocate("generic")
err = qemuBalloon.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -1710,9 +1600,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
return "", err
}
- devBus, devAddr, multi = allocateBusAddr("generic")
+ devBus, devAddr, multi = bus.allocate("generic")
err = qemuRNG.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -1721,9 +1611,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
return "", err
}
- devBus, devAddr, multi = allocateBusAddr("generic")
+ devBus, devAddr, multi = bus.allocate("generic")
err = qemuKeyboard.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -1732,9 +1622,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
return "", err
}
- devBus, devAddr, multi = allocateBusAddr("generic")
+ devBus, devAddr, multi = bus.allocate("generic")
err = qemuTablet.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -1743,9 +1633,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
return "", err
}
- devBus, devAddr, multi = allocateBusAddr("generic")
+ devBus, devAddr, multi = bus.allocate("generic")
err = qemuVsock.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -1756,9 +1646,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
return "", err
}
- devBus, devAddr, multi = allocateBusAddr("generic")
+ devBus, devAddr, multi = bus.allocate("generic")
err = qemuSerial.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -1769,9 +1659,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
return "", err
}
- devBus, devAddr, multi = allocateBusAddr("")
+ devBus, devAddr, multi = bus.allocate("")
err = qemuSCSI.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -1780,9 +1670,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
return "", err
}
- devBus, devAddr, multi = allocateBusAddr("9p")
+ devBus, devAddr, multi = bus.allocate("9p")
err = qemuDriveConfig.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -1793,9 +1683,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
return "", err
}
- devBus, devAddr, multi = allocateBusAddr("")
+ devBus, devAddr, multi = bus.allocate("")
err = qemuGPU.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -1821,7 +1711,7 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
if drive.TargetPath == "/" {
err = vm.addRootDriveConfig(sb, bootIndexes, drive)
} else if drive.FSType == "9p" {
- err = vm.addDriveDirConfig(sb, bus, allocateBusAddr, fdFiles, &agentMounts, drive)
+ err = vm.addDriveDirConfig(sb, bus, fdFiles, &agentMounts, drive)
} else {
err = vm.addDriveConfig(sb, bootIndexes, drive)
}
@@ -1833,7 +1723,7 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
// Add network device.
if len(runConf.NetworkInterface) > 0 {
- err = vm.addNetDevConfig(sb, bus, allocateBusAddr, bootIndexes, runConf.NetworkInterface, fdFiles)
+ err = vm.addNetDevConfig(sb, bus, bootIndexes, runConf.NetworkInterface, fdFiles)
if err != nil {
return "", err
}
@@ -1990,7 +1880,7 @@ func (vm *qemu) addRootDriveConfig(sb *strings.Builder, bootIndexes map[string]i
}
// addDriveDirConfig adds the qemu config required for adding a supplementary drive directory share.
-func (vm *qemu) addDriveDirConfig(sb *strings.Builder, bus string, allocateBusAddr func(group string) (string, string, bool), fdFiles *[]string, agentMounts *[]instancetype.VMAgentMount, driveConf deviceConfig.MountEntryItem) error {
+func (vm *qemu) addDriveDirConfig(sb *strings.Builder, bus *qemuBus, fdFiles *[]string, agentMounts *[]instancetype.VMAgentMount, driveConf deviceConfig.MountEntryItem) error {
mountTag := fmt.Sprintf("lxd_%s", driveConf.DevName)
agentMount := instancetype.VMAgentMount{
@@ -2008,12 +1898,12 @@ func (vm *qemu) addDriveDirConfig(sb *strings.Builder, bus string, allocateBusAd
// Record the 9p mount for the agent.
*agentMounts = append(*agentMounts, agentMount)
- devBus, devAddr, multi := allocateBusAddr("9p")
+ devBus, devAddr, multi := bus.allocate("9p")
// For read only shares, do not use proxy.
if shared.StringInSlice("ro", driveConf.Opts) {
return qemuDriveDir.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -2028,7 +1918,7 @@ func (vm *qemu) addDriveDirConfig(sb *strings.Builder, bus string, allocateBusAd
// Only use proxy for writable shares.
proxyFD := vm.addFileDescriptor(fdFiles, driveConf.DevPath)
return qemuDriveDir.Execute(sb, map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devBus": devBus,
"devAddr": devAddr,
"multifunction": multi,
@@ -2078,7 +1968,7 @@ func (vm *qemu) addDriveConfig(sb *strings.Builder, bootIndexes map[string]int,
}
// addNetDevConfig adds the qemu config required for adding a network device.
-func (vm *qemu) addNetDevConfig(sb *strings.Builder, bus string, allocateBusAddr func(group string) (string, string, bool), bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem, fdFiles *[]string) error {
+func (vm *qemu) addNetDevConfig(sb *strings.Builder, bus *qemuBus, bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem, fdFiles *[]string) error {
var devName, nicName, devHwaddr, pciSlotName string
for _, nicItem := range nicConfig {
if nicItem.Key == "devName" {
@@ -2094,7 +1984,7 @@ func (vm *qemu) addNetDevConfig(sb *strings.Builder, bus string, allocateBusAddr
var tpl *template.Template
tplFields := map[string]interface{}{
- "bus": bus,
+ "bus": bus.name,
"devName": devName,
"devHwaddr": devHwaddr,
"bootIndex": bootIndexes[devName],
@@ -2126,7 +2016,7 @@ func (vm *qemu) addNetDevConfig(sb *strings.Builder, bus string, allocateBusAddr
tpl = qemuNetdevPhysical
}
- devBus, devAddr, multi := allocateBusAddr("")
+ devBus, devAddr, multi := bus.allocate("")
tplFields["devBus"] = devBus
tplFields["devAddr"] = devAddr
tplFields["multifunction"] = multi
diff --git a/lxd/instance/drivers/driver_qemu_bus.go b/lxd/instance/drivers/driver_qemu_bus.go
new file mode 100644
index 0000000000..c38e47d572
--- /dev/null
+++ b/lxd/instance/drivers/driver_qemu_bus.go
@@ -0,0 +1,142 @@
+package drivers
+
+import (
+ "fmt"
+ "strings"
+)
+
+type qemuBusEntry struct {
+ bridgeDev int // Device number on the root bridge.
+ bridgeFn int // Function number on the root bridge.
+
+ dev string // Existing device name.
+ fn int // Function number on the existing device.
+}
+
+type qemuBus struct {
+ name string // Bus type.
+ sb *strings.Builder // String builder to use.
+
+ portNum int // Next available port/chassis on the bridge.
+ devNum int // Next available device number on the bridge.
+
+ rootPort *qemuBusEntry // Current root port.
+
+ entries map[string]*qemuBusEntry // Map of qemuBusEntry for a particular shared device.
+}
+
+func (a *qemuBus) allocateRoot() *qemuBusEntry {
+ if a.rootPort == nil {
+ a.rootPort = &qemuBusEntry{
+ bridgeDev: a.devNum,
+ }
+ a.devNum++
+ } else {
+ if a.rootPort.bridgeFn == 7 {
+ a.rootPort.bridgeFn = 0
+ a.rootPort.bridgeDev = a.devNum
+ a.devNum++
+ } else {
+ a.rootPort.bridgeFn++
+ }
+ }
+
+ return a.rootPort
+}
+
+// allocate() does any needed port allocation and returns the bus name,
+// address and whether the device needs to be configured as multi-function.
+//
+// The group parameter allows for grouping devices together as a single
+// multi-function device. It automatically keeps track of the number of
+// functions already used and will allocate a new device as needed.
+func (a *qemuBus) allocate(group string) (string, string, bool) {
+ if a.name == "ccw" {
+ return "", "", false
+ }
+
+ // Find a device group if specified.
+ var p *qemuBusEntry
+ if group != "" {
+ var ok bool
+ p, ok = a.entries[group]
+ if ok {
+ // Check if group is full.
+ if p.fn == 7 {
+ p.fn = 0
+ if a.name == "pci" {
+ p.bridgeDev = a.devNum
+ a.devNum++
+ } else if a.name == "pcie" {
+ r := a.allocateRoot()
+ p.bridgeDev = r.bridgeDev
+ p.bridgeFn = r.bridgeFn
+ }
+ } else {
+ p.fn++
+ }
+ } else {
+ // Create a new group.
+ p = &qemuBusEntry{}
+
+ if a.name == "pci" {
+ p.bridgeDev = a.devNum
+ a.devNum++
+ } else if a.name == "pcie" {
+ r := a.allocateRoot()
+ p.bridgeDev = r.bridgeDev
+ p.bridgeFn = r.bridgeFn
+ }
+
+ a.entries[group] = p
+ }
+ } else {
+ // Create a new temporary group.
+ p = &qemuBusEntry{}
+
+ if a.name == "pci" {
+ p.bridgeDev = a.devNum
+ a.devNum++
+ } else if a.name == "pcie" {
+ r := a.allocateRoot()
+ p.bridgeDev = r.bridgeDev
+ p.bridgeFn = r.bridgeFn
+ }
+ }
+
+ multi := p.fn == 0 && group != ""
+
+ if a.name == "pci" {
+ return "pci.0", fmt.Sprintf("%x.%d", p.bridgeDev, p.fn), multi
+ }
+
+ if a.name == "pcie" {
+ if p.fn == 0 {
+ qemuPCIe.Execute(a.sb, map[string]interface{}{
+ "index": a.portNum,
+ "addr": fmt.Sprintf("%x.%d", p.bridgeDev, p.bridgeFn),
+ "multifunction": p.bridgeFn == 0,
+ })
+ p.dev = fmt.Sprintf("qemu_pcie%d", a.portNum)
+ a.portNum++
+ }
+
+ return p.dev, fmt.Sprintf("00.%d", p.fn), multi
+ }
+
+ return "", "", false
+}
+
+func qemuNewBus(name string, sb *strings.Builder) *qemuBus {
+ a := &qemuBus{
+ name: name,
+ sb: sb,
+
+ portNum: 0, // No PCIe ports are used in the default config.
+ devNum: 1, // Address 0 is used by the DRAM controller.
+
+ entries: map[string]*qemuBusEntry{},
+ }
+
+ return a
+}
More information about the lxc-devel
mailing list