[lxc-devel] [lxd/master] Initial PCIe cleanup
stgraber on Github
lxc-bot at linuxcontainers.org
Wed Jun 10 18:27:24 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200610/904b807d/attachment-0001.bin>
-------------- next part --------------
From 135cdf1e71cb7a085392443fe7a700273bb9598b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 9 Jun 2020 19:14:45 -0400
Subject: [PATCH 1/3] lxd/vm: Move to separate devices
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
One template per device type.
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/instance/drivers/driver_qemu.go | 56 +++++++++++++++++--
lxd/instance/drivers/driver_qemu_templates.go | 40 +++++++------
2 files changed, 76 insertions(+), 20 deletions(-)
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index b12bb68730..2620fec50e 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -1552,15 +1552,34 @@ func (vm *qemu) generateQemuConfigFile(devConfs []*deviceConfig.RunConfig, fdFil
var sb *strings.Builder = &strings.Builder{}
err := qemuBase.Execute(sb, map[string]interface{}{
- "architecture": vm.architectureName,
- "ringbufSizeBytes": qmp.RingbufSize,
- "spicePath": vm.spicePath(),
+ "architecture": vm.architectureName,
+ "spicePath": vm.spicePath(),
})
if err != nil {
return "", err
}
// Now add the dynamic parts of the config.
+ err = vm.addSerialConfig(sb)
+ if err != nil {
+ return "", err
+ }
+
+ err = vm.addSCSIConfig(sb)
+ if err != nil {
+ return "", err
+ }
+
+ err = vm.addBalloonConfig(sb)
+ if err != nil {
+ return "", err
+ }
+
+ err = vm.addRNGConfig(sb)
+ if err != nil {
+ return "", err
+ }
+
err = vm.addMemoryConfig(sb)
if err != nil {
return "", err
@@ -1691,7 +1710,36 @@ func (vm *qemu) addVsockConfig(sb *strings.Builder) error {
})
}
-// addVGAConfig adds the qemu config required for setting up the host->VM vsock socket.
+// addSerialConfig adds the qemu config required for the identifier serial device.
+func (vm *qemu) addSerialConfig(sb *strings.Builder) error {
+ return qemuSerial.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ "ringbufSizeBytes": qmp.RingbufSize,
+ })
+}
+
+// addSCSIConfig adds the qemu config required for SCSI devices.
+func (vm *qemu) addSCSIConfig(sb *strings.Builder) error {
+ return qemuSCSI.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ })
+}
+
+// addBalloonConfig adds the qemu config required for the memory balloon drive.
+func (vm *qemu) addBalloonConfig(sb *strings.Builder) error {
+ return qemuBalloon.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ })
+}
+
+// addRNGConfig adds the qemu config required for the random number generator.
+func (vm *qemu) addRNGConfig(sb *strings.Builder) error {
+ return qemuRNG.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ })
+}
+
+// addVGAConfig adds the qemu config required for the default graphics card.
func (vm *qemu) addVGAConfig(sb *strings.Builder, chassisIndex int, gpuIndex int) error {
return qemuVGA.Execute(sb, map[string]interface{}{
"architecture": vm.architectureName,
diff --git a/lxd/instance/drivers/driver_qemu_templates.go b/lxd/instance/drivers/driver_qemu_templates.go
index be1b982057..bce1d2cc8d 100644
--- a/lxd/instance/drivers/driver_qemu_templates.go
+++ b/lxd/instance/drivers/driver_qemu_templates.go
@@ -41,6 +41,24 @@ value = "1"
[boot-opts]
strict = "on"
+# Console
+[chardev "console"]
+backend = "pty"
+
+# Graphical console
+[spice]
+unix = "on"
+addr = "{{.spicePath}}"
+disable-ticketing = "on"
+`))
+
+var qemuMemory = template.Must(template.New("qemuMemory").Parse(`
+# Memory
+[memory]
+size = "{{.memSizeBytes}}B"
+`))
+
+var qemuSerial = template.Must(template.New("qemuSerial").Parse(`
# LXD serial identifier
[device]
driver = "virtio-serial"
@@ -53,7 +71,9 @@ chardev = "vserial"
[chardev "vserial"]
backend = "ringbuf"
size = "{{.ringbufSizeBytes}}B"
+`))
+var qemuSCSI = template.Must(template.New("qemuSCSI").Parse(`
# SCSI controller
{{- if eq .architecture "x86_64" "aarch64" }}
[device "qemu_pcie1"]
@@ -77,7 +97,9 @@ addr = "0x0"
{{- else}}
driver = "virtio-scsi-ccw"
{{- end}}
+`))
+var qemuBalloon = template.Must(template.New("qemuBalloon").Parse(`
# Balloon driver
{{- if eq .architecture "x86_64" "aarch64" }}
[device "qemu_pcie2"]
@@ -100,7 +122,9 @@ addr = "0x0"
{{- else}}
driver = "virtio-balloon-ccw"
{{- end}}
+`))
+var qemuRNG = template.Must(template.New("qemuRNG").Parse(`
# Random number generator
[object "qemu_rng"]
qom-type = "rng-random"
@@ -128,22 +152,6 @@ addr = "0x0"
{{- else}}
driver = "virtio-rng-ccw"
{{- end}}
-
-# Console
-[chardev "console"]
-backend = "pty"
-
-# Graphical console
-[spice]
-unix = "on"
-addr = "{{.spicePath}}"
-disable-ticketing = "on"
-`))
-
-var qemuMemory = template.Must(template.New("qemuMemory").Parse(`
-# Memory
-[memory]
-size = "{{.memSizeBytes}}B"
`))
var qemuVsock = template.Must(template.New("qemuVsock").Parse(`
From 8bca065126076e3fddae7b8d025ab1f9bd60d787 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 10 Jun 2020 10:50:03 -0400
Subject: [PATCH 2/3] lxd/vm: Remove tiny wrapper functions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/instance/drivers/driver_qemu.go | 125 +++++-------------
lxd/instance/drivers/driver_qemu_templates.go | 2 +
2 files changed, 38 insertions(+), 89 deletions(-)
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 2620fec50e..d7b0ab383e 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -1560,22 +1560,31 @@ func (vm *qemu) generateQemuConfigFile(devConfs []*deviceConfig.RunConfig, fdFil
}
// Now add the dynamic parts of the config.
- err = vm.addSerialConfig(sb)
+ err = qemuSerial.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ "ringbufSizeBytes": qmp.RingbufSize,
+ })
if err != nil {
return "", err
}
- err = vm.addSCSIConfig(sb)
+ err = qemuSCSI.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ })
if err != nil {
return "", err
}
- err = vm.addBalloonConfig(sb)
+ err = qemuBalloon.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ })
if err != nil {
return "", err
}
- err = vm.addRNGConfig(sb)
+ err = qemuRNG.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ })
if err != nil {
return "", err
}
@@ -1590,17 +1599,24 @@ func (vm *qemu) generateQemuConfigFile(devConfs []*deviceConfig.RunConfig, fdFil
return "", err
}
- err = vm.addFirmwareConfig(sb)
- if err != nil {
- return "", err
- }
+ err = qemuDriveFirmware.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ "roPath": filepath.Join(vm.ovmfPath(), "OVMF_CODE.fd"),
+ "nvramPath": vm.nvramPath(),
+ })
- err = vm.addVsockConfig(sb)
+ err = qemuVsock.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ "vsockID": vm.vsockID(),
+ })
if err != nil {
return "", err
}
- err = vm.addMonitorConfig(sb)
+ err = qemuControlSocket.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ "path": vm.monitorPath(),
+ })
if err != nil {
return "", err
}
@@ -1613,14 +1629,22 @@ func (vm *qemu) generateQemuConfigFile(devConfs []*deviceConfig.RunConfig, fdFil
gpuIndex := 0
chassisIndex := 5 // Internal devices defined in the templates use indexes 1-4.
- err = vm.addConfDriveConfig(sb, diskIndex)
+ err = qemuDriveConfig.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ "path": filepath.Join(vm.Path(), "config"),
+ "diskIndex": diskIndex,
+ })
if err != nil {
return "", err
}
diskIndex++ // The config drive is a 9p device which uses a PCIe function so increment index.
// GPU for console.
- err = vm.addVGAConfig(sb, chassisIndex, gpuIndex)
+ err = qemuVGA.Execute(sb, map[string]interface{}{
+ "architecture": vm.architectureName,
+ "chassisIndex": chassisIndex,
+ "gpuIndex": gpuIndex,
+ })
if err != nil {
return "", err
}
@@ -1702,52 +1726,6 @@ func (vm *qemu) addMemoryConfig(sb *strings.Builder) error {
})
}
-// addVsockConfig adds the qemu config required for setting up the host->VM vsock socket.
-func (vm *qemu) addVsockConfig(sb *strings.Builder) error {
- return qemuVsock.Execute(sb, map[string]interface{}{
- "architecture": vm.architectureName,
- "vsockID": vm.vsockID(),
- })
-}
-
-// addSerialConfig adds the qemu config required for the identifier serial device.
-func (vm *qemu) addSerialConfig(sb *strings.Builder) error {
- return qemuSerial.Execute(sb, map[string]interface{}{
- "architecture": vm.architectureName,
- "ringbufSizeBytes": qmp.RingbufSize,
- })
-}
-
-// addSCSIConfig adds the qemu config required for SCSI devices.
-func (vm *qemu) addSCSIConfig(sb *strings.Builder) error {
- return qemuSCSI.Execute(sb, map[string]interface{}{
- "architecture": vm.architectureName,
- })
-}
-
-// addBalloonConfig adds the qemu config required for the memory balloon drive.
-func (vm *qemu) addBalloonConfig(sb *strings.Builder) error {
- return qemuBalloon.Execute(sb, map[string]interface{}{
- "architecture": vm.architectureName,
- })
-}
-
-// addRNGConfig adds the qemu config required for the random number generator.
-func (vm *qemu) addRNGConfig(sb *strings.Builder) error {
- return qemuRNG.Execute(sb, map[string]interface{}{
- "architecture": vm.architectureName,
- })
-}
-
-// addVGAConfig adds the qemu config required for the default graphics card.
-func (vm *qemu) addVGAConfig(sb *strings.Builder, chassisIndex int, gpuIndex int) error {
- return qemuVGA.Execute(sb, map[string]interface{}{
- "architecture": vm.architectureName,
- "chassisIndex": chassisIndex,
- "gpuIndex": gpuIndex,
- })
-}
-
// addCPUConfig adds the qemu config required for setting the number of virtualised CPUs.
func (vm *qemu) addCPUConfig(sb *strings.Builder) error {
// Default to a single core.
@@ -1820,37 +1798,6 @@ func (vm *qemu) addCPUConfig(sb *strings.Builder) error {
return qemuCPU.Execute(sb, ctx)
}
-// addMonitorConfig adds the qemu config required for setting up the host side VM monitor device.
-func (vm *qemu) addMonitorConfig(sb *strings.Builder) error {
- return qemuControlSocket.Execute(sb, map[string]interface{}{
- "architecture": vm.architectureName,
- "path": vm.monitorPath(),
- })
-}
-
-// addFirmwareConfig adds the qemu config required for adding a secure boot compatible EFI firmware.
-func (vm *qemu) addFirmwareConfig(sb *strings.Builder) error {
- // UEFI only on x86_64 and aarch64.
- if !shared.IntInSlice(vm.architecture, []int{osarch.ARCH_64BIT_INTEL_X86, osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN}) {
- return nil
- }
-
- return qemuDriveFirmware.Execute(sb, map[string]interface{}{
- "architecture": vm.architectureName,
- "roPath": filepath.Join(vm.ovmfPath(), "OVMF_CODE.fd"),
- "nvramPath": vm.nvramPath(),
- })
-}
-
-// addConfDriveConfig adds the qemu config required for adding the config drive.
-func (vm *qemu) addConfDriveConfig(sb *strings.Builder, diskIndex int) error {
- return qemuDriveConfig.Execute(sb, map[string]interface{}{
- "architecture": vm.architectureName,
- "path": filepath.Join(vm.Path(), "config"),
- "diskIndex": diskIndex,
- })
-}
-
// addFileDescriptor adds a file path to the list of files to open and pass file descriptor to qemu.
// Returns the file descriptor number that qemu will receive.
func (vm *qemu) addFileDescriptor(fdFiles *[]string, filePath string) int {
diff --git a/lxd/instance/drivers/driver_qemu_templates.go b/lxd/instance/drivers/driver_qemu_templates.go
index bce1d2cc8d..cb1cc10d69 100644
--- a/lxd/instance/drivers/driver_qemu_templates.go
+++ b/lxd/instance/drivers/driver_qemu_templates.go
@@ -246,6 +246,7 @@ mode = "control"
`))
var qemuDriveFirmware = template.Must(template.New("qemuDriveFirmware").Parse(`
+{{if eq .architecture "x86_64" "aarch64" -}}
# Firmware (read only)
[drive]
file = "{{.roPath}}"
@@ -260,6 +261,7 @@ file = "{{.nvramPath}}"
if = "pflash"
format = "raw"
unit = "1"
+{{- end }}
`))
// Devices use "qemu_" prefix indicating that this is a internally named device.
From 3f3dcd9e6563d4efa2970b8f73cebb4e4e876305 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 10 Jun 2020 10:50:23 -0400
Subject: [PATCH 3/3] lxd/vm: Per-architecture bus type
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/instance/drivers/driver_qemu.go | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index d7b0ab383e..44f9ee0987 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -692,7 +692,7 @@ func (vm *qemu) Start(stateful bool) error {
}
// Get qemu configuration.
- qemuBinary, err := vm.qemuArchConfig()
+ qemuBinary, qemuBus, err := vm.qemuArchConfig()
if err != nil {
op.Done(err)
return err
@@ -701,7 +701,7 @@ func (vm *qemu) Start(stateful bool) error {
// Define a set of files to open and pass their file descriptors to qemu command.
fdFiles := make([]string, 0)
- confFile, err := vm.generateQemuConfigFile(devConfs, &fdFiles)
+ confFile, err := vm.generateQemuConfigFile(qemuBus, devConfs, &fdFiles)
if err != nil {
op.Done(err)
return err
@@ -998,18 +998,18 @@ func (vm *qemu) setupNvram() error {
return nil
}
-func (vm *qemu) qemuArchConfig() (string, error) {
+func (vm *qemu) qemuArchConfig() (string, string, error) {
if vm.architecture == osarch.ARCH_64BIT_INTEL_X86 {
- return "qemu-system-x86_64", nil
+ return "qemu-system-x86_64", "pcie", nil
} else if vm.architecture == osarch.ARCH_64BIT_ARMV8_LITTLE_ENDIAN {
- return "qemu-system-aarch64", nil
+ return "qemu-system-aarch64", "pcie", nil
} else if vm.architecture == osarch.ARCH_64BIT_POWERPC_LITTLE_ENDIAN {
- return "qemu-system-ppc64", nil
+ return "qemu-system-ppc64", "pci", nil
} else if vm.architecture == osarch.ARCH_64BIT_S390_BIG_ENDIAN {
- return "qemu-system-s390x", nil
+ return "qemu-system-s390x", "ccw", nil
}
- return "", fmt.Errorf("Architecture isn't supported for virtual machines")
+ return "", "", fmt.Errorf("Architecture isn't supported for virtual machines")
}
// deviceVolatileGetFunc returns a function that retrieves a named device's volatile config and
@@ -1548,7 +1548,7 @@ func (vm *qemu) deviceBootPriorities() (map[string]int, error) {
// generateQemuConfigFile writes the qemu config file and returns its location.
// It writes the config file inside the VM's log path.
-func (vm *qemu) generateQemuConfigFile(devConfs []*deviceConfig.RunConfig, fdFiles *[]string) (string, error) {
+func (vm *qemu) generateQemuConfigFile(bus string, devConfs []*deviceConfig.RunConfig, fdFiles *[]string) (string, error) {
var sb *strings.Builder = &strings.Builder{}
err := qemuBase.Execute(sb, map[string]interface{}{
More information about the lxc-devel
mailing list