[lxc-devel] [lxd/master] Support virtual GPUs
monstermunchkin on Github
lxc-bot at linuxcontainers.org
Fri Nov 6 13:01:18 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20201106/d048f2be/attachment.bin>
-------------- next part --------------
From b9ebdef70cac8153b509a06032e0ff98acc794c7 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Fri, 6 Nov 2020 09:43:23 +0100
Subject: [PATCH 1/6] shared: Allow volatile uuid config keys
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
shared/instance.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/shared/instance.go b/shared/instance.go
index 06541b4539..5234f5c751 100644
--- a/shared/instance.go
+++ b/shared/instance.go
@@ -305,7 +305,7 @@ func ConfigKeyChecker(key string) (func(value string) error, error) {
return validate.IsAny, nil
}
- if strings.HasSuffix(key, "vm.uuid") {
+ if strings.HasSuffix(key, ".uuid") {
return validate.IsAny, nil
}
From 36ff22e7c69079e78d86727c23cb6de7e42791b7 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Fri, 6 Nov 2020 09:45:15 +0100
Subject: [PATCH 2/6] lxd/instance/drivers: Support vgpu in qemu template
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/instance/drivers/driver_qemu_templates.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lxd/instance/drivers/driver_qemu_templates.go b/lxd/instance/drivers/driver_qemu_templates.go
index c0b1cc27d8..68c1927f67 100644
--- a/lxd/instance/drivers/driver_qemu_templates.go
+++ b/lxd/instance/drivers/driver_qemu_templates.go
@@ -528,10 +528,14 @@ addr = "{{.devAddr}}"
{{if eq .bus "ccw" -}}
driver = "vfio-ccw"
{{- end}}
+{{- if ne .vgpu "" -}}
+sysfsdev = "/sys/bus/mdev/devices/{{.vgpu}}"
+{{- else}}
host = "{{.pciSlotName}}"
{{if .vga -}}
x-vga = "on"
{{- end }}
+{{- end }}
{{if .multifunction -}}
multifunction = "on"
{{- end }}
From 240d577d3402de0d03dea285a3e2fa8b24dbceec Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Fri, 6 Nov 2020 09:46:46 +0100
Subject: [PATCH 3/6] lxd/instance/drivers: Support vgpu in VMs
---
lxd/instance/drivers/driver_qemu.go | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 41174d770e..cdb753ebdf 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -2364,12 +2364,14 @@ func (vm *qemu) addNetDevConfig(sb *strings.Builder, bus *qemuBus, bootIndexes m
// addGPUDevConfig adds the qemu config required for adding a GPU device.
func (vm *qemu) addGPUDevConfig(sb *strings.Builder, bus *qemuBus, gpuConfig []deviceConfig.RunConfigItem) error {
- var devName, pciSlotName string
+ var devName, pciSlotName, vgpu string
for _, gpuItem := range gpuConfig {
if gpuItem.Key == "devName" {
devName = gpuItem.Value
} else if gpuItem.Key == "pciSlotName" {
pciSlotName = gpuItem.Value
+ } else if gpuItem.Key == "vgpu" {
+ vgpu = gpuItem.Value
}
}
@@ -2386,6 +2388,7 @@ func (vm *qemu) addGPUDevConfig(sb *strings.Builder, bus *qemuBus, gpuConfig []d
"devName": devName,
"pciSlotName": pciSlotName,
"vga": vgaMode,
+ "vgpu": vgpu,
}
// Add main GPU device in VGA mode to qemu config.
@@ -2394,8 +2397,14 @@ func (vm *qemu) addGPUDevConfig(sb *strings.Builder, bus *qemuBus, gpuConfig []d
return err
}
- // Add any other related IOMMU VFs as generic PCI devices.
- iommuGroupPath := filepath.Join("/sys/bus/pci/devices", pciSlotName, "iommu_group", "devices")
+ var iommuGroupPath string
+
+ if vgpu != "" {
+ iommuGroupPath = filepath.Join("/sys/bus/mdev/devices", vgpu, "iommu_group", "devices")
+ } else {
+ // Add any other related IOMMU VFs as generic PCI devices.
+ iommuGroupPath = filepath.Join("/sys/bus/pci/devices", pciSlotName, "iommu_group", "devices")
+ }
if shared.PathExists(iommuGroupPath) {
// Extract parent slot name by removing any virtual function ID.
From 593eb27c3a14ae7d533c3874aaeeb7b9c8e6669d Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Fri, 6 Nov 2020 09:54:36 +0100
Subject: [PATCH 4/6] lxd/device: Support virtual GPUs
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
lxd/device/gpu.go | 135 +++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 128 insertions(+), 7 deletions(-)
diff --git a/lxd/device/gpu.go b/lxd/device/gpu.go
index 9c7724077b..d2a2ecfed5 100644
--- a/lxd/device/gpu.go
+++ b/lxd/device/gpu.go
@@ -9,6 +9,7 @@ import (
"strconv"
"strings"
+ "github.com/google/uuid"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
@@ -17,10 +18,12 @@ import (
"github.com/lxc/lxd/lxd/instance/instancetype"
"github.com/lxc/lxd/lxd/resources"
"github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/logger"
"github.com/lxc/lxd/shared/validate"
)
const gpuDRIDevPath = "/dev/dri"
+const gpuVFIODevPath = "/dev/vfio"
// Non-card devices such as {/dev/nvidiactl, /dev/nvidia-uvm, ...}
type nvidiaNonCardDevice struct {
@@ -47,6 +50,7 @@ func (d *gpu) validateConfig(instConf instance.ConfigReader) error {
"uid": unixValidUserID,
"gid": unixValidUserID,
"mode": unixValidOctalFileMode,
+ "mdev": validate.IsAny,
}
err := d.config.Validate(rules)
@@ -90,6 +94,62 @@ func (d *gpu) validateEnvironment() error {
return nil
}
+func (d *gpu) createVirtualGPU() error {
+ gpus, err := resources.GetGPU()
+ if err != nil {
+ return err
+ }
+
+ for _, gpu := range gpus.Cards {
+ // Skip any cards that don't match the vendorid, pci or productid settings (if specified).
+ if (d.config["vendorid"] != "" && gpu.VendorID != d.config["vendorid"]) ||
+ (d.config["pci"] != "" && gpu.PCIAddress != d.config["pci"]) ||
+ (d.config["productid"] != "" && gpu.ProductID != d.config["productid"]) {
+ continue
+ }
+
+ foundMdev := false
+
+ for mdev := range gpu.Mdev {
+ if d.config["mdev"] == mdev {
+ foundMdev = true
+ break
+ }
+ }
+
+ if !foundMdev {
+ return fmt.Errorf("Invalid mdev %q", d.config["mdev"])
+ }
+
+ // Check if the vgpu exists before creating it.
+ v := d.volatileGet()
+
+ if v["vgpu.uuid"] != "" && shared.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s/%s", gpu.PCIAddress, v["vgpu.uuid"])) {
+ return nil
+ }
+
+ // Create the virtual gpu
+ devUUID, err := uuid.NewUUID()
+ if err != nil {
+ return errors.Wrap(err, "Failed to generate UUID")
+ }
+
+ err = ioutil.WriteFile(filepath.Join(fmt.Sprintf("/sys/bus/pci/devices/%s/mdev_supported_types/%s/create", gpu.PCIAddress, d.config["mdev"])), []byte(devUUID.String()), 200)
+ if err != nil {
+ return errors.Wrapf(err, "Failed to create virtual gpu %q", devUUID.String())
+ }
+
+ err = d.volatileSet(map[string]string{"vgpu.uuid": devUUID.String()})
+ if err != nil {
+ return err
+ }
+
+ break
+ }
+
+ return nil
+}
+
// Start is run when the device is added to the container.
func (d *gpu) Start() (*deviceConfig.RunConfig, error) {
err := d.validateEnvironment()
@@ -97,6 +157,13 @@ func (d *gpu) Start() (*deviceConfig.RunConfig, error) {
return nil, err
}
+ if d.config["mdev"] != "" {
+ err = d.createVirtualGPU()
+ if err != nil {
+ return nil, err
+ }
+ }
+
if d.inst.Type() == instancetype.VM {
return d.startVM()
}
@@ -128,7 +195,38 @@ func (d *gpu) startContainer() (*deviceConfig.RunConfig, error) {
if gpu.DRM != nil && (d.config["id"] == "" || fmt.Sprintf("%d", gpu.DRM.ID) == d.config["id"]) {
found = true
- if gpu.DRM.CardName != "" && gpu.DRM.CardDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.CardName)) {
+ if d.config["mdev"] != "" {
+ v := d.volatileGet()
+
+ // Get iommu group
+ vgpuPath := fmt.Sprintf("/sys/bus/pci/devices/%s/%s/iommu_group", gpu.PCIAddress, v["vgpu.uuid"])
+
+ link, err := os.Readlink(vgpuPath)
+ if err != nil {
+ return nil, err
+ }
+
+ iommuGroup := filepath.Base(link)
+ path := filepath.Join(gpuVFIODevPath, iommuGroup)
+
+ // Get major and minor numbers
+ var stat unix.Stat_t
+
+ err = unix.Stat(path, &stat)
+ if err != nil {
+ return nil, err
+ }
+
+ major := uint32(stat.Rdev / 256)
+ minor := uint32(stat.Rdev % 256)
+
+ err = unixDeviceSetupCharNum(d.state, d.inst.DevicesPath(), "unix", d.name, d.config, major, minor, path, false, &runConf)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ if d.config["mdev"] == "" && gpu.DRM.CardName != "" && gpu.DRM.CardDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.CardName)) {
path := filepath.Join(gpuDRIDevPath, gpu.DRM.CardName)
major, minor, err := d.deviceNumStringToUint32(gpu.DRM.CardDevice)
if err != nil {
@@ -141,7 +239,7 @@ func (d *gpu) startContainer() (*deviceConfig.RunConfig, error) {
}
}
- if gpu.DRM.RenderName != "" && gpu.DRM.RenderDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.RenderName)) {
+ if d.config["mdev"] == "" && gpu.DRM.RenderName != "" && gpu.DRM.RenderDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.RenderName)) {
path := filepath.Join(gpuDRIDevPath, gpu.DRM.RenderName)
major, minor, err := d.deviceNumStringToUint32(gpu.DRM.RenderDevice)
if err != nil {
@@ -154,7 +252,7 @@ func (d *gpu) startContainer() (*deviceConfig.RunConfig, error) {
}
}
- if gpu.DRM.ControlName != "" && gpu.DRM.ControlDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.ControlName)) {
+ if d.config["mdev"] == "" && gpu.DRM.ControlName != "" && gpu.DRM.ControlDevice != "" && shared.PathExists(filepath.Join(gpuDRIDevPath, gpu.DRM.ControlName)) {
path := filepath.Join(gpuDRIDevPath, gpu.DRM.ControlName)
major, minor, err := d.deviceNumStringToUint32(gpu.DRM.ControlDevice)
if err != nil {
@@ -256,9 +354,11 @@ func (d *gpu) startVM() (*deviceConfig.RunConfig, error) {
saveData["last_state.pci.slot.name"] = pciDev.SlotName
saveData["last_state.pci.driver"] = pciDev.Driver
- err = d.pciDeviceDriverOverrideIOMMU(pciDev, "vfio-pci", false)
- if err != nil {
- return nil, errors.Wrapf(err, "Failed to override IOMMU group driver")
+ if d.config["mdev"] == "" {
+ err = d.pciDeviceDriverOverrideIOMMU(pciDev, "vfio-pci", false)
+ if err != nil {
+ return nil, errors.Wrapf(err, "Failed to override IOMMU group driver")
+ }
}
runConf.GPUDevice = append(runConf.GPUDevice,
@@ -267,6 +367,15 @@ func (d *gpu) startVM() (*deviceConfig.RunConfig, error) {
{Key: "pciSlotName", Value: saveData["last_state.pci.slot.name"]},
}...)
+ if d.config["mdev"] != "" {
+ v := d.volatileGet()
+
+ saveData["vgpu.uuid"] = v["vgpu.uuid"]
+
+ runConf.GPUDevice = append(runConf.GPUDevice,
+ deviceConfig.RunConfigItem{Key: "vgpu", Value: v["vgpu.uuid"]})
+ }
+
err = d.volatileSet(saveData)
if err != nil {
return nil, err
@@ -349,10 +458,22 @@ func (d *gpu) postStop() error {
defer d.volatileSet(map[string]string{
"last_state.pci.slot.name": "",
"last_state.pci.driver": "",
+ "vgpu.uuid": "",
})
v := d.volatileGet()
+ if v["vgpu.uuid"] != "" {
+ path := fmt.Sprintf("/sys/bus/mdev/devices/%s", v["vgpu.uuid"])
+
+ if shared.PathExists(path) {
+ err := ioutil.WriteFile(filepath.Join(path, "remove"), []byte("1\n"), 0200)
+ if err != nil {
+ logger.Debugf("Failed to remove vgpu %q", v["vgpu.uuid"])
+ }
+ }
+ }
+
if d.inst.Type() == instancetype.Container {
// Remove host files for this device.
err := unixDeviceDeleteFiles(d.state, d.inst.DevicesPath(), "unix", d.name, "")
@@ -362,7 +483,7 @@ func (d *gpu) postStop() error {
}
// If VM physical pass through, unbind from vfio-pci and bind back to host driver.
- if d.inst.Type() == instancetype.VM && v["last_state.pci.slot.name"] != "" {
+ if d.inst.Type() == instancetype.VM && v["last_state.pci.slot.name"] != "" && v["vgpu.uuid"] == "" {
pciDev := pciDevice{
Driver: "vfio-pci",
SlotName: v["last_state.pci.slot.name"],
From 531ca6adbb80027ff433ac6c6e4ab4c1c79622d3 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Fri, 6 Nov 2020 10:21:28 +0100
Subject: [PATCH 5/6] doc: Document mdev config key
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
doc/instances.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/doc/instances.md b/doc/instances.md
index d6809ff115..cf6b89a687 100644
--- a/doc/instances.md
+++ b/doc/instances.md
@@ -763,6 +763,7 @@ pci | string | - | no | The pci address of the
uid | int | 0 | no | UID of the device owner in the instance (container only)
gid | int | 0 | no | GID of the device owner in the instance (container only)
mode | int | 0660 | no | Mode of the device in the instance (container only)
+mdev | string | - | no | The mdev type to use (e.g. i915-GVTg_V5_4)
### Type: proxy
From c8520ff699b1fe0ab9603e0bfa58de09a5c343dc Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Fri, 6 Nov 2020 13:45:29 +0100
Subject: [PATCH 6/6] api: Add virtual_gpu
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
doc/api-extensions.md | 4 ++++
shared/version/api.go | 1 +
2 files changed, 5 insertions(+)
diff --git a/doc/api-extensions.md b/doc/api-extensions.md
index 555e83d8c6..cc42d755e0 100644
--- a/doc/api-extensions.md
+++ b/doc/api-extensions.md
@@ -1222,3 +1222,7 @@ This introduces the `tpm` device type.
This introduces `rebase` as a value for zfs.clone\_copy causing LXD to
track down any "image" dataset in the ancestry line and then perform
send/receive on top of that.
+
+## virtual\_gpu
+This adds support for virtual GPUs. It introduces the `mdev` config key for GPU devices which takes
+a supported mdev type, e.g. i915-GVTg_V5_4.
diff --git a/shared/version/api.go b/shared/version/api.go
index 2c986fa5f0..d827894d2d 100644
--- a/shared/version/api.go
+++ b/shared/version/api.go
@@ -235,6 +235,7 @@ var APIExtensions = []string{
"network_ovn_external_routes_remove",
"tpm_device_type",
"storage_zfs_clone_copy_rebase",
+ "virtual_gpu",
}
// APIExtensionsCount returns the number of available API extensions.
More information about the lxc-devel
mailing list