[lxc-devel] [lxd/master] lxd/vm: Move bus allocator to own file

stgraber on Github lxc-bot at linuxcontainers.org
Thu Jun 11 23:47:06 UTC 2020


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 354 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200611/2f43f1ec/attachment.bin>
-------------- next part --------------
From 69598929e8eac6447019dd5087ec38534fec72b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 11 Jun 2020 19:46:38 -0400
Subject: [PATCH] lxd/vm: Move bus allocator to own file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: St├ęphane Graber <stgraber at ubuntu.com>
---
 lxd/instance/drivers/driver_qemu.go     | 170 +++++-------------------
 lxd/instance/drivers/driver_qemu_bus.go | 142 ++++++++++++++++++++
 2 files changed, 172 insertions(+), 140 deletions(-)
 create mode 100644 lxd/instance/drivers/driver_qemu_bus.go

diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index c2a81f479f..7c12346a98 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -1548,7 +1548,7 @@ func (vm *qemu) deviceBootPriorities() (map[string]int, error) {
 
 // generateQemuConfigFile writes the qemu config file and returns its location.
 // It writes the config file inside the VM's log path.
-func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunConfig, fdFiles *[]string) (string, error) {
+func (vm *qemu) generateQemuConfigFile(busName string, devConfs []*deviceConfig.RunConfig, fdFiles *[]string) (string, error) {
 	var sb *strings.Builder = &strings.Builder{}
 
 	err := qemuBase.Execute(sb, map[string]interface{}{
@@ -1585,123 +1585,13 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 		return "", err
 	}
 
-	// allocateBusAddress is used to create any needed root ports and provide
-	// the bus and address that should be used by a device. It supports
-	// automatically setting up multi-function devices and optimize their use.
-	portNum := 0
-	devNum := 1
-
-	type entry struct {
-		bridgeDev int // Device number on the root bridge.
-		bridgeFn  int // Function number on the root bridge.
-
-		dev string // Existing device name.
-		fn  int    // Function number on the existing device.
-	}
-	entries := map[string]*entry{}
-
-	var rootPort *entry
-	allocateRootPort := func() *entry {
-		if rootPort == nil {
-			rootPort = &entry{
-				bridgeDev: devNum,
-			}
-			devNum++
-		} else {
-			if rootPort.bridgeFn == 7 {
-				rootPort.bridgeFn = 0
-				rootPort.bridgeDev = devNum
-				devNum++
-			} else {
-				rootPort.bridgeFn++
-			}
-		}
-
-		return rootPort
-	}
-
-	allocateBusAddr := func(group string) (string, string, bool) {
-		// FIXME: Need to figure out if ccw needs any bus logic.
-		if bus == "ccw" {
-			return "", "", false
-		}
-
-		// Find a device group if specified.
-		var p *entry
-		if group != "" {
-			var ok bool
-			p, ok = entries[group]
-			if ok {
-				// Check if group is full.
-				if p.fn == 7 {
-					p.fn = 0
-					if bus == "pci" {
-						p.bridgeDev = devNum
-						devNum++
-					} else if bus == "pcie" {
-						r := allocateRootPort()
-						p.bridgeDev = r.bridgeDev
-						p.bridgeFn = r.bridgeFn
-					}
-				} else {
-					p.fn++
-				}
-			} else {
-				// Create a new group.
-				p = &entry{}
-
-				if bus == "pci" {
-					p.bridgeDev = devNum
-					devNum++
-				} else if bus == "pcie" {
-					r := allocateRootPort()
-					p.bridgeDev = r.bridgeDev
-					p.bridgeFn = r.bridgeFn
-				}
-
-				entries[group] = p
-			}
-		} else {
-			// Create a new temporary group.
-			p = &entry{}
-
-			if bus == "pci" {
-				p.bridgeDev = devNum
-				devNum++
-			} else if bus == "pcie" {
-				r := allocateRootPort()
-				p.bridgeDev = r.bridgeDev
-				p.bridgeFn = r.bridgeFn
-			}
-		}
-
-		multi := p.fn == 0 && group != ""
-
-		if bus == "pci" {
-			return "pci.0", fmt.Sprintf("%x.%d", p.bridgeDev, p.fn), multi
-		}
-
-		if bus == "pcie" {
-			if p.fn == 0 {
-				qemuPCIe.Execute(sb, map[string]interface{}{
-					"index":         portNum,
-					"addr":          fmt.Sprintf("%x.%d", p.bridgeDev, p.bridgeFn),
-					"multifunction": p.bridgeFn == 0,
-				})
-				p.dev = fmt.Sprintf("qemu_pcie%d", portNum)
-				portNum++
-			}
-
-			return p.dev, fmt.Sprintf("00.%d", p.fn), multi
-		}
-
-		return "", "", false
-	}
+	// Setup the bus allocator.
+	bus := qemuNewBus(busName, sb)
 
 	// Now add the fixed set of devices.
-	devBus, devAddr, multi := allocateBusAddr("generic")
+	devBus, devAddr, multi := bus.allocate("generic")
 	err = qemuBalloon.Execute(sb, map[string]interface{}{
-		"bus":           bus,
+		"bus":           bus.name,
 		"devBus":        devBus,
 		"devAddr":       devAddr,
 		"multifunction": multi,
@@ -1710,9 +1600,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 		return "", err
 	}
 
-	devBus, devAddr, multi = allocateBusAddr("generic")
+	devBus, devAddr, multi = bus.allocate("generic")
 	err = qemuRNG.Execute(sb, map[string]interface{}{
-		"bus":           bus,
+		"bus":           bus.name,
 		"devBus":        devBus,
 		"devAddr":       devAddr,
 		"multifunction": multi,
@@ -1721,9 +1611,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 		return "", err
 	}
 
-	devBus, devAddr, multi = allocateBusAddr("generic")
+	devBus, devAddr, multi = bus.allocate("generic")
 	err = qemuKeyboard.Execute(sb, map[string]interface{}{
-		"bus":           bus,
+		"bus":           bus.name,
 		"devBus":        devBus,
 		"devAddr":       devAddr,
 		"multifunction": multi,
@@ -1732,9 +1622,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 		return "", err
 	}
 
-	devBus, devAddr, multi = allocateBusAddr("generic")
+	devBus, devAddr, multi = bus.allocate("generic")
 	err = qemuTablet.Execute(sb, map[string]interface{}{
-		"bus":           bus,
+		"bus":           bus.name,
 		"devBus":        devBus,
 		"devAddr":       devAddr,
 		"multifunction": multi,
@@ -1743,9 +1633,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 		return "", err
 	}
 
-	devBus, devAddr, multi = allocateBusAddr("generic")
+	devBus, devAddr, multi = bus.allocate("generic")
 	err = qemuVsock.Execute(sb, map[string]interface{}{
-		"bus":           bus,
+		"bus":           bus.name,
 		"devBus":        devBus,
 		"devAddr":       devAddr,
 		"multifunction": multi,
@@ -1756,9 +1646,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 		return "", err
 	}
 
-	devBus, devAddr, multi = allocateBusAddr("generic")
+	devBus, devAddr, multi = bus.allocate("generic")
 	err = qemuSerial.Execute(sb, map[string]interface{}{
-		"bus":           bus,
+		"bus":           bus.name,
 		"devBus":        devBus,
 		"devAddr":       devAddr,
 		"multifunction": multi,
@@ -1769,9 +1659,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 		return "", err
 	}
 
-	devBus, devAddr, multi = allocateBusAddr("")
+	devBus, devAddr, multi = bus.allocate("")
 	err = qemuSCSI.Execute(sb, map[string]interface{}{
-		"bus":           bus,
+		"bus":           bus.name,
 		"devBus":        devBus,
 		"devAddr":       devAddr,
 		"multifunction": multi,
@@ -1780,9 +1670,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 		return "", err
 	}
 
-	devBus, devAddr, multi = allocateBusAddr("9p")
+	devBus, devAddr, multi = bus.allocate("9p")
 	err = qemuDriveConfig.Execute(sb, map[string]interface{}{
-		"bus":           bus,
+		"bus":           bus.name,
 		"devBus":        devBus,
 		"devAddr":       devAddr,
 		"multifunction": multi,
@@ -1793,9 +1683,9 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 		return "", err
 	}
 
-	devBus, devAddr, multi = allocateBusAddr("")
+	devBus, devAddr, multi = bus.allocate("")
 	err = qemuGPU.Execute(sb, map[string]interface{}{
-		"bus":           bus,
+		"bus":           bus.name,
 		"devBus":        devBus,
 		"devAddr":       devAddr,
 		"multifunction": multi,
@@ -1821,7 +1711,7 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 				if drive.TargetPath == "/" {
 					err = vm.addRootDriveConfig(sb, bootIndexes, drive)
 				} else if drive.FSType == "9p" {
-					err = vm.addDriveDirConfig(sb, bus, allocateBusAddr, fdFiles, &agentMounts, drive)
+					err = vm.addDriveDirConfig(sb, bus, fdFiles, &agentMounts, drive)
 				} else {
 					err = vm.addDriveConfig(sb, bootIndexes, drive)
 				}
@@ -1833,7 +1723,7 @@ func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunC
 
 		// Add network device.
 		if len(runConf.NetworkInterface) > 0 {
-			err = vm.addNetDevConfig(sb, bus, allocateBusAddr, bootIndexes, runConf.NetworkInterface, fdFiles)
+			err = vm.addNetDevConfig(sb, bus, bootIndexes, runConf.NetworkInterface, fdFiles)
 			if err != nil {
 				return "", err
 			}
@@ -1990,7 +1880,7 @@ func (vm *qemu) addRootDriveConfig(sb *strings.Builder, bootIndexes map[string]i
 }
 
 // addDriveDirConfig adds the qemu config required for adding a supplementary drive directory share.
-func (vm *qemu) addDriveDirConfig(sb *strings.Builder, bus string, allocateBusAddr func(group string) (string, string, bool), fdFiles *[]string, agentMounts *[]instancetype.VMAgentMount, driveConf deviceConfig.MountEntryItem) error {
+func (vm *qemu) addDriveDirConfig(sb *strings.Builder, bus *qemuBus, fdFiles *[]string, agentMounts *[]instancetype.VMAgentMount, driveConf deviceConfig.MountEntryItem) error {
 	mountTag := fmt.Sprintf("lxd_%s", driveConf.DevName)
 
 	agentMount := instancetype.VMAgentMount{
@@ -2008,12 +1898,12 @@ func (vm *qemu) addDriveDirConfig(sb *strings.Builder, bus string, allocateBusAd
 	// Record the 9p mount for the agent.
 	*agentMounts = append(*agentMounts, agentMount)
 
-	devBus, devAddr, multi := allocateBusAddr("9p")
+	devBus, devAddr, multi := bus.allocate("9p")
 
 	// For read only shares, do not use proxy.
 	if shared.StringInSlice("ro", driveConf.Opts) {
 		return qemuDriveDir.Execute(sb, map[string]interface{}{
-			"bus":           bus,
+			"bus":           bus.name,
 			"devBus":        devBus,
 			"devAddr":       devAddr,
 			"multifunction": multi,
@@ -2028,7 +1918,7 @@ func (vm *qemu) addDriveDirConfig(sb *strings.Builder, bus string, allocateBusAd
 	// Only use proxy for writable shares.
 	proxyFD := vm.addFileDescriptor(fdFiles, driveConf.DevPath)
 	return qemuDriveDir.Execute(sb, map[string]interface{}{
-		"bus":           bus,
+		"bus":           bus.name,
 		"devBus":        devBus,
 		"devAddr":       devAddr,
 		"multifunction": multi,
@@ -2078,7 +1968,7 @@ func (vm *qemu) addDriveConfig(sb *strings.Builder, bootIndexes map[string]int,
 }
 
 // addNetDevConfig adds the qemu config required for adding a network device.
-func (vm *qemu) addNetDevConfig(sb *strings.Builder, bus string, allocateBusAddr func(group string) (string, string, bool), bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem, fdFiles *[]string) error {
+func (vm *qemu) addNetDevConfig(sb *strings.Builder, bus *qemuBus, bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem, fdFiles *[]string) error {
 	var devName, nicName, devHwaddr, pciSlotName string
 	for _, nicItem := range nicConfig {
 		if nicItem.Key == "devName" {
@@ -2094,7 +1984,7 @@ func (vm *qemu) addNetDevConfig(sb *strings.Builder, bus string, allocateBusAddr
 
 	var tpl *template.Template
 	tplFields := map[string]interface{}{
-		"bus":       bus,
+		"bus":       bus.name,
 		"devName":   devName,
 		"devHwaddr": devHwaddr,
 		"bootIndex": bootIndexes[devName],
@@ -2126,7 +2016,7 @@ func (vm *qemu) addNetDevConfig(sb *strings.Builder, bus string, allocateBusAddr
 		tpl = qemuNetdevPhysical
 	}
 
-	devBus, devAddr, multi := allocateBusAddr("")
+	devBus, devAddr, multi := bus.allocate("")
 	tplFields["devBus"] = devBus
 	tplFields["devAddr"] = devAddr
 	tplFields["multifunction"] = multi
diff --git a/lxd/instance/drivers/driver_qemu_bus.go b/lxd/instance/drivers/driver_qemu_bus.go
new file mode 100644
index 0000000000..c38e47d572
--- /dev/null
+++ b/lxd/instance/drivers/driver_qemu_bus.go
@@ -0,0 +1,142 @@
+package drivers
+
+import (
+	"fmt"
+	"strings"
+)
+
+type qemuBusEntry struct {
+	bridgeDev int // Device number on the root bridge.
+	bridgeFn  int // Function number on the root bridge.
+
+	dev string // Existing device name.
+	fn  int    // Function number on the existing device.
+}
+
+type qemuBus struct {
+	name string           // Bus type.
+	sb   *strings.Builder // String builder to use.
+
+	portNum int // Next available port/chassis on the bridge.
+	devNum  int // Next available device number on the bridge.
+
+	rootPort *qemuBusEntry // Current root port.
+
+	entries map[string]*qemuBusEntry // Map of qemuBusEntry for a particular shared device.
+}
+
+func (a *qemuBus) allocateRoot() *qemuBusEntry {
+	if a.rootPort == nil {
+		a.rootPort = &qemuBusEntry{
+			bridgeDev: a.devNum,
+		}
+		a.devNum++
+	} else {
+		if a.rootPort.bridgeFn == 7 {
+			a.rootPort.bridgeFn = 0
+			a.rootPort.bridgeDev = a.devNum
+			a.devNum++
+		} else {
+			a.rootPort.bridgeFn++
+		}
+	}
+
+	return a.rootPort
+}
+
+// allocate() does any needed port allocation and  returns the bus name,
+// address and whether the device needs to be configured as multi-function.
+//
+// The group parameter allows for grouping devices together as a single
+// multi-function device. It automatically keeps track of the number of
+// functions already used and will allocate a new device as needed.
+func (a *qemuBus) allocate(group string) (string, string, bool) {
+	if a.name == "ccw" {
+		return "", "", false
+	}
+
+	// Find a device group if specified.
+	var p *qemuBusEntry
+	if group != "" {
+		var ok bool
+		p, ok = a.entries[group]
+		if ok {
+			// Check if group is full.
+			if p.fn == 7 {
+				p.fn = 0
+				if a.name == "pci" {
+					p.bridgeDev = a.devNum
+					a.devNum++
+				} else if a.name == "pcie" {
+					r := a.allocateRoot()
+					p.bridgeDev = r.bridgeDev
+					p.bridgeFn = r.bridgeFn
+				}
+			} else {
+				p.fn++
+			}
+		} else {
+			// Create a new group.
+			p = &qemuBusEntry{}
+
+			if a.name == "pci" {
+				p.bridgeDev = a.devNum
+				a.devNum++
+			} else if a.name == "pcie" {
+				r := a.allocateRoot()
+				p.bridgeDev = r.bridgeDev
+				p.bridgeFn = r.bridgeFn
+			}
+
+			a.entries[group] = p
+		}
+	} else {
+		// Create a new temporary group.
+		p = &qemuBusEntry{}
+
+		if a.name == "pci" {
+			p.bridgeDev = a.devNum
+			a.devNum++
+		} else if a.name == "pcie" {
+			r := a.allocateRoot()
+			p.bridgeDev = r.bridgeDev
+			p.bridgeFn = r.bridgeFn
+		}
+	}
+
+	multi := p.fn == 0 && group != ""
+
+	if a.name == "pci" {
+		return "pci.0", fmt.Sprintf("%x.%d", p.bridgeDev, p.fn), multi
+	}
+
+	if a.name == "pcie" {
+		if p.fn == 0 {
+			qemuPCIe.Execute(a.sb, map[string]interface{}{
+				"index":         a.portNum,
+				"addr":          fmt.Sprintf("%x.%d", p.bridgeDev, p.bridgeFn),
+				"multifunction": p.bridgeFn == 0,
+			})
+			p.dev = fmt.Sprintf("qemu_pcie%d", a.portNum)
+			a.portNum++
+		}
+
+		return p.dev, fmt.Sprintf("00.%d", p.fn), multi
+	}
+
+	return "", "", false
+}
+
+func qemuNewBus(name string, sb *strings.Builder) *qemuBus {
+	a := &qemuBus{
+		name: name,
+		sb:   sb,
+
+		portNum: 0, // No PCIe ports are used in the default config.
+		devNum:  1, // Address 0 is used by the DRAM controller.
+
+		entries: map[string]*qemuBusEntry{},
+	}
+
+	return a
+}


More information about the lxc-devel mailing list