[lxc-devel] [lxd/master] Device GPU

tomponline on Github lxc-bot at linuxcontainers.org
Thu Aug 8 10:45:26 UTC 2019


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 372 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190808/c132ade3/attachment-0001.bin>
-------------- next part --------------
From fc07030f5467162fbc3045842a70f063307e3eed Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:31:29 +0100
Subject: [PATCH 1/9] devices: Removes gpu related code moved to device package

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/device/gpu.go | 556 ++++++++++++++++++++++++++++++++++++++++++++++
 lxd/devices.go    | 478 ---------------------------------------
 2 files changed, 556 insertions(+), 478 deletions(-)
 create mode 100644 lxd/device/gpu.go

diff --git a/lxd/device/gpu.go b/lxd/device/gpu.go
new file mode 100644
index 0000000000..d176930542
--- /dev/null
+++ b/lxd/device/gpu.go
@@ -0,0 +1,556 @@
+package device
+
+import (
+	"fmt"
+
+	"github.com/lxc/lxd/lxd/device/config"
+	"github.com/lxc/lxd/lxd/instance"
+	"github.com/lxc/lxd/shared"
+)
+
+type gpu struct {
+	deviceCommon
+}
+
+// /dev/dri/card0. If we detect that vendor == nvidia, then nvidia will contain
+// the corresponding nvidia car, e.g. {/dev/dri/card1 to /dev/nvidia1}.
+type gpuDevice struct {
+	// DRM node information
+	id    string
+	path  string
+	major int
+	minor int
+
+	// Device information
+	vendorID    string
+	vendorName  string
+	productID   string
+	productName string
+	numaNode    uint64
+
+	// If related devices have the same PCI address as the GPU we should
+	// mount them all. Meaning if we detect /dev/dri/card0,
+	// /dev/dri/controlD64, and /dev/dri/renderD128 with the same PCI
+	// address, then they should all be made available in the container.
+	pci           string
+	driver        string
+	driverVersion string
+
+	// NVIDIA specific handling
+	isNvidia bool
+	nvidia   nvidiaGpuCard
+}
+
+// /dev/nvidia[0-9]+
+type nvidiaGpuCard struct {
+	path  string
+	major int
+	minor int
+	id    string
+
+	nvrmVersion  string
+	cudaVersion  string
+	model        string
+	brand        string
+	uuid         string
+	architecture string
+}
+
+// {/dev/nvidiactl, /dev/nvidia-uvm, ...}
+type nvidiaGpuDevice struct {
+	isCard bool
+	path   string
+	major  int
+	minor  int
+}
+
+// validateConfig checks the supplied config for correctness.
+func (d *gpu) validateConfig() error {
+	if d.instance.Type() != instance.TypeContainer {
+		return ErrUnsupportedDevType
+	}
+
+	rules := map[string]func(string) error{
+		"vendorid":  shared.IsAny,
+		"productid": shared.IsAny,
+		"id":        shared.IsAny,
+		"pci":       shared.IsAny,
+		"uid":       shared.IsUnixUserID,
+		"gid":       shared.IsUnixUserID,
+		"mode":      shared.IsOctalFileMode,
+	}
+
+	err := config.ValidateDevice(rules, d.config)
+	if err != nil {
+		return err
+	}
+
+	if d.config["pci"] != "" && (d.config["id"] != "" || d.config["productid"] != "" || d.config["vendorid"] != "") {
+		return fmt.Errorf("Cannot use id, productid or vendorid when pci is set")
+	}
+
+	if d.config["id"] != "" && (d.config["pci"] != "" || d.config["productid"] != "" || d.config["vendorid"] != "") {
+		return fmt.Errorf("Cannot use pci, productid or vendorid when id is set")
+	}
+
+	return nil
+}
+
+// validateEnvironment checks the runtime environment for correctness.
+func (d *gpu) validateEnvironment() error {
+	if d.config["pci"] != "" && !shared.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s", d.config["pci"])) {
+		return fmt.Errorf("Invalid PCI address (no device found): %s", d.config["pci"])
+	}
+
+	return nil
+}
+
+// Start is run when the device is added to the container.
+func (d *gpu) Start() (*RunConfig, error) {
+	err := d.validateEnvironment()
+	if err != nil {
+		return nil, err
+	}
+
+	allGpus := d.deviceWantsAllGPUs(d.config)
+	gpus, nvidiaDevices, err := d.deviceLoadGpu(allGpus)
+	if err != nil {
+		return nil, err
+	}
+
+	sawNvidia := false
+	found := false
+	for _, gpu := range gpus {
+		if (d.config["vendorid"] != "" && gpu.vendorID != d.config["vendorid"]) ||
+			(d.config["pci"] != "" && gpu.pci != d.config["pci"]) ||
+			(d.config["productid"] != "" && gpu.productID != d.config["productid"]) ||
+			(d.config["id"] != "" && gpu.id != d.config["id"]) {
+			continue
+		}
+
+		found = true
+
+		err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, false)
+		if err != nil {
+			logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
+			return err
+		}
+
+		if !gpu.isNvidia {
+			continue
+		}
+
+		if gpu.nvidia.path != "" {
+			err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path, false)
+			if err != nil {
+				logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
+				return err
+			}
+		} else if !allGpus {
+			errMsg := fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
+			logger.Errorf("%s", errMsg)
+			return errMsg
+		}
+
+		sawNvidia = true
+	}
+
+	if sawNvidia {
+		for _, gpu := range nvidiaDevices {
+			if shared.IsTrue(c.expandedConfig["nvidia.runtime"]) {
+				if !gpu.isCard {
+					continue
+				}
+			}
+
+			if c.deviceExistsInDevicesFolder(k, gpu.path) {
+				continue
+			}
+
+			err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, false)
+			if err != nil {
+				logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
+				return err
+			}
+		}
+	}
+
+	if !found {
+		msg := "Failed to detect requested GPU device"
+		logger.Error(msg)
+		return fmt.Errorf(msg)
+	}
+
+	return nil, nil
+}
+
+func (d *gpu) configureUnixDeviceNum(name string, config config.Device, major int, minor int, path string, defaultMode bool, runConf *RunConfig) error {
+	paths, err := UnixCreateDevice(s, nil, devicesPath, devPrefix, dummyDevice, false)
+	if err != nil {
+		return err
+	}
+	devPath := paths[0]
+
+	// Instruct liblxc to perform the mount.
+	runConf.Mounts = append(runConf.Mounts, MountEntryItem{
+		DevPath:    devPath,
+		TargetPath: relDestPath,
+		FSType:     "none",
+		Opts:       []string{"bind", "create=file"},
+	})
+
+	// Add the new device cgroup rule.
+	dType, dMajor, dMinor, err := UnixGetDeviceAttributes(devPath)
+	if err != nil {
+		return err
+	}
+
+	// Instruct liblxc to setup the cgroup rule.
+	runConf.CGroups = append(runConf.CGroups, RunConfigItem{
+		Key:   "devices.allow",
+		Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor),
+	})
+}
+
+// deviceWantsAllGPUs whether the LXD device wants to passthrough all GPUs on the host.
+func (d *gpu) deviceWantsAllGPUs(m map[string]string) bool {
+	return m["vendorid"] == "" && m["productid"] == "" && m["id"] == "" && m["pci"] == ""
+}
+
+// deviceLoadGpu probes the system for information about the availble GPUs.
+func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
+	const DRM_PATH = "/sys/class/drm/"
+	var gpus []gpuDevice
+	var nvidiaDevices []nvidiaGpuDevice
+	var cards []cardIds
+
+	// Load NVIDIA information (if available)
+	var nvidiaContainer *nvidiaContainerInfo
+
+	_, err := exec.LookPath("nvidia-container-cli")
+	if err == nil {
+		out, err := shared.RunCommand("nvidia-container-cli", "info", "--csv")
+		if err == nil {
+			r := csv.NewReader(strings.NewReader(out))
+			r.FieldsPerRecord = -1
+
+			nvidiaContainer = &nvidiaContainerInfo{}
+			nvidiaContainer.Cards = map[string]*nvidiaContainerCardInfo{}
+			line := 0
+			for {
+				record, err := r.Read()
+				if err == io.EOF {
+					break
+				}
+				line += 1
+
+				if err != nil {
+					continue
+				}
+
+				if line == 2 && len(record) >= 2 {
+					nvidiaContainer.NVRMVersion = record[0]
+					nvidiaContainer.CUDAVersion = record[1]
+				} else if line >= 4 {
+					nvidiaContainer.Cards[record[5]] = &nvidiaContainerCardInfo{
+						DeviceIndex:  record[0],
+						DeviceMinor:  record[1],
+						Model:        record[2],
+						Brand:        record[3],
+						UUID:         record[4],
+						PCIAddress:   record[5],
+						Architecture: record[6],
+					}
+				}
+			}
+		}
+	}
+
+	// Load PCI database
+	pciDB, err := pcidb.New()
+	if err != nil {
+		pciDB = nil
+	}
+
+	// Get the list of DRM devices
+	ents, err := ioutil.ReadDir(DRM_PATH)
+	if err != nil {
+		// No GPUs
+		if os.IsNotExist(err) {
+			return nil, nil, nil
+		}
+
+		return nil, nil, err
+	}
+
+	// Get the list of cards
+	devices := []string{}
+	for _, ent := range ents {
+		dev, err := filepath.EvalSymlinks(fmt.Sprintf("%s/%s/device", DRM_PATH, ent.Name()))
+		if err != nil {
+			continue
+		}
+
+		if !shared.StringInSlice(dev, devices) {
+			devices = append(devices, dev)
+		}
+	}
+
+	isNvidia := false
+	for _, device := range devices {
+		// The pci address == the name of the directory. So let's use
+		// this cheap way of retrieving it.
+		pciAddr := filepath.Base(device)
+
+		// Make sure that we are dealing with a GPU by looking whether
+		// the "drm" subfolder exists.
+		drm := filepath.Join(device, "drm")
+		drmEnts, err := ioutil.ReadDir(drm)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+		}
+
+		// Retrieve vendor ID.
+		vendorIdPath := filepath.Join(device, "vendor")
+		vendorId, err := ioutil.ReadFile(vendorIdPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+		}
+
+		// Retrieve device ID.
+		productIdPath := filepath.Join(device, "device")
+		productId, err := ioutil.ReadFile(productIdPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+		}
+
+		// Retrieve node ID
+		numaPath := fmt.Sprintf(filepath.Join(device, "numa_node"))
+		numaNode := uint64(0)
+		if shared.PathExists(numaPath) {
+			numaID, err := shared.ParseNumberFromFile(numaPath)
+			if err != nil {
+				continue
+			}
+
+			if numaID > 0 {
+				numaNode = uint64(numaID)
+			}
+		}
+
+		// Retrieve driver
+		driver := ""
+		driverVersion := ""
+		driverPath := filepath.Join(device, "driver")
+		if shared.PathExists(driverPath) {
+			target, err := os.Readlink(driverPath)
+			if err != nil {
+				continue
+			}
+
+			driver = filepath.Base(target)
+
+			out, err := ioutil.ReadFile(filepath.Join(driverPath, "module", "version"))
+			if err == nil {
+				driverVersion = strings.TrimSpace(string(out))
+			} else {
+				uname, err := shared.Uname()
+				if err != nil {
+					continue
+				}
+				driverVersion = uname.Release
+			}
+		}
+
+		// Store all associated subdevices, e.g. controlD64, renderD128.
+		// The name of the directory == the last part of the
+		// /dev/dri/controlD64 path. So drmEnt.Name() will give us
+		// controlD64.
+		for _, drmEnt := range drmEnts {
+			vendorTmp := strings.TrimSpace(string(vendorId))
+			productTmp := strings.TrimSpace(string(productId))
+			vendorTmp = strings.TrimPrefix(vendorTmp, "0x")
+			productTmp = strings.TrimPrefix(productTmp, "0x")
+			tmpGpu := gpuDevice{
+				pci:           pciAddr,
+				vendorID:      vendorTmp,
+				productID:     productTmp,
+				numaNode:      numaNode,
+				driver:        driver,
+				driverVersion: driverVersion,
+				path:          filepath.Join("/dev/dri", drmEnt.Name()),
+			}
+
+			// Fill vendor and product names
+			if pciDB != nil {
+				vendor, ok := pciDB.Vendors[tmpGpu.vendorID]
+				if ok {
+					tmpGpu.vendorName = vendor.Name
+
+					for _, product := range vendor.Products {
+						if product.ID == tmpGpu.productID {
+							tmpGpu.productName = product.Name
+							break
+						}
+					}
+				}
+			}
+
+			majMinPath := filepath.Join(drm, drmEnt.Name(), "dev")
+			majMinByte, err := ioutil.ReadFile(majMinPath)
+			if err != nil {
+				if os.IsNotExist(err) {
+					continue
+				}
+			}
+
+			majMin := strings.TrimSpace(string(majMinByte))
+			majMinSlice := strings.Split(string(majMin), ":")
+			if len(majMinSlice) != 2 {
+				continue
+			}
+
+			majorInt, err := strconv.Atoi(majMinSlice[0])
+			if err != nil {
+				continue
+			}
+
+			minorInt, err := strconv.Atoi(majMinSlice[1])
+			if err != nil {
+				continue
+			}
+
+			tmpGpu.major = majorInt
+			tmpGpu.minor = minorInt
+
+			isCard, err := regexp.MatchString("^card[0-9]+", drmEnt.Name())
+			if err != nil {
+				continue
+			}
+
+			// Find matching /dev/nvidia* entry for /dev/dri/card*
+			if tmpGpu.isNvidiaGpu() && isCard {
+				if !isNvidia {
+					isNvidia = true
+				}
+				tmpGpu.isNvidia = true
+
+				if !all {
+					minor, err := findNvidiaMinor(tmpGpu.pci)
+					if err == nil {
+						nvidiaPath := "/dev/nvidia" + minor
+						stat := unix.Stat_t{}
+						err = unix.Stat(nvidiaPath, &stat)
+						if err != nil {
+							if os.IsNotExist(err) {
+								continue
+							}
+
+							return nil, nil, err
+						}
+
+						tmpGpu.nvidia.path = nvidiaPath
+						tmpGpu.nvidia.major = shared.Major(stat.Rdev)
+						tmpGpu.nvidia.minor = shared.Minor(stat.Rdev)
+						tmpGpu.nvidia.id = strconv.Itoa(tmpGpu.nvidia.minor)
+
+						if nvidiaContainer != nil {
+							tmpGpu.nvidia.nvrmVersion = nvidiaContainer.NVRMVersion
+							tmpGpu.nvidia.cudaVersion = nvidiaContainer.CUDAVersion
+							nvidiaInfo, ok := nvidiaContainer.Cards[tmpGpu.pci]
+							if !ok {
+								nvidiaInfo, ok = nvidiaContainer.Cards[fmt.Sprintf("0000%v", tmpGpu.pci)]
+							}
+							if ok {
+								tmpGpu.nvidia.brand = nvidiaInfo.Brand
+								tmpGpu.nvidia.model = nvidiaInfo.Model
+								tmpGpu.nvidia.uuid = nvidiaInfo.UUID
+								tmpGpu.nvidia.architecture = nvidiaInfo.Architecture
+							}
+						}
+					}
+				}
+			}
+
+			if isCard {
+				// If it is a card it's minor number will be its id.
+				tmpGpu.id = strconv.Itoa(minorInt)
+				tmp := cardIds{
+					id:  tmpGpu.id,
+					pci: tmpGpu.pci,
+				}
+
+				cards = append(cards, tmp)
+			}
+
+			gpus = append(gpus, tmpGpu)
+		}
+	}
+
+	// We detected a Nvidia card, so let's collect all other nvidia devices
+	// that are not /dev/nvidia[0-9]+.
+	if isNvidia {
+		nvidiaEnts, err := ioutil.ReadDir("/dev")
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil, nil, err
+			}
+		}
+
+		validNvidia, err := regexp.Compile(`^nvidia[^0-9]+`)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		for _, nvidiaEnt := range nvidiaEnts {
+			if all {
+				if !strings.HasPrefix(nvidiaEnt.Name(), "nvidia") {
+					continue
+				}
+			} else {
+				if !validNvidia.MatchString(nvidiaEnt.Name()) {
+					continue
+				}
+			}
+
+			nvidiaPath := filepath.Join("/dev", nvidiaEnt.Name())
+			stat := unix.Stat_t{}
+			err = unix.Stat(nvidiaPath, &stat)
+			if err != nil {
+				continue
+			}
+
+			tmpNividiaGpu := nvidiaGpuDevice{
+				isCard: !validNvidia.MatchString(nvidiaEnt.Name()),
+				path:   nvidiaPath,
+				major:  shared.Major(stat.Rdev),
+				minor:  shared.Minor(stat.Rdev),
+			}
+
+			nvidiaDevices = append(nvidiaDevices, tmpNividiaGpu)
+		}
+	}
+
+	// Since we'll give users to ability to specify and id we need to group
+	// devices on the same PCI that belong to the same card by id.
+	for _, card := range cards {
+		for i := 0; i < len(gpus); i++ {
+			if gpus[i].pci == card.pci {
+				gpus[i].id = card.id
+			}
+		}
+	}
+
+	return gpus, nvidiaDevices, nil
+}
+
+// Stop is run when the device is removed from the instance.
+func (d *gpu) Stop() (*RunConfig, error) {
+	return nil, nil
+}
diff --git a/lxd/devices.go b/lxd/devices.go
index b0d987e68e..dd28b25728 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -4,22 +4,17 @@ import (
 	"bufio"
 	"bytes"
 	"crypto/rand"
-	"encoding/csv"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"math/big"
 	"os"
-	"os/exec"
 	"path"
 	"path/filepath"
-	"regexp"
 	"sort"
 	"strconv"
 	"strings"
 	"unsafe"
 
-	"github.com/jaypipes/pcidb"
 	"golang.org/x/sys/unix"
 
 	"github.com/lxc/lxd/lxd/device"
@@ -66,479 +61,6 @@ type usbDevice struct {
 	ueventLen   int
 }
 
-// /dev/nvidia[0-9]+
-type nvidiaGpuCard struct {
-	path  string
-	major int
-	minor int
-	id    string
-
-	nvrmVersion  string
-	cudaVersion  string
-	model        string
-	brand        string
-	uuid         string
-	architecture string
-}
-
-// {/dev/nvidiactl, /dev/nvidia-uvm, ...}
-type nvidiaGpuDevice struct {
-	isCard bool
-	path   string
-	major  int
-	minor  int
-}
-
-// Nvidia container info
-type nvidiaContainerInfo struct {
-	Cards       map[string]*nvidiaContainerCardInfo
-	NVRMVersion string
-	CUDAVersion string
-}
-
-type nvidiaContainerCardInfo struct {
-	DeviceIndex  string
-	DeviceMinor  string
-	Model        string
-	Brand        string
-	UUID         string
-	PCIAddress   string
-	Architecture string
-}
-
-// /dev/dri/card0. If we detect that vendor == nvidia, then nvidia will contain
-// the corresponding nvidia car, e.g. {/dev/dri/card1 to /dev/nvidia1}.
-type gpuDevice struct {
-	// DRM node information
-	id    string
-	path  string
-	major int
-	minor int
-
-	// Device information
-	vendorID    string
-	vendorName  string
-	productID   string
-	productName string
-	numaNode    uint64
-
-	// If related devices have the same PCI address as the GPU we should
-	// mount them all. Meaning if we detect /dev/dri/card0,
-	// /dev/dri/controlD64, and /dev/dri/renderD128 with the same PCI
-	// address, then they should all be made available in the container.
-	pci           string
-	driver        string
-	driverVersion string
-
-	// NVIDIA specific handling
-	isNvidia bool
-	nvidia   nvidiaGpuCard
-}
-
-func (g *gpuDevice) isNvidiaGpu() bool {
-	return strings.EqualFold(g.vendorID, "10de")
-}
-
-type cardIds struct {
-	id  string
-	pci string
-}
-
-// Fallback for old drivers which don't provide "Device Minor:"
-func findNvidiaMinorOld() (string, error) {
-	var minor string
-
-	// For now, just handle most common case (single nvidia card)
-	ents, err := ioutil.ReadDir("/dev")
-	if err != nil {
-		return "", err
-	}
-
-	rp := regexp.MustCompile("^nvidia([0-9]+)$")
-	for _, ent := range ents {
-		matches := rp.FindStringSubmatch(ent.Name())
-		if matches == nil {
-			continue
-		}
-
-		if minor != "" {
-			return "", fmt.Errorf("No device minor index detected, and more than one NVIDIA card present")
-		}
-		minor = matches[1]
-	}
-
-	if minor == "" {
-		return "", fmt.Errorf("No device minor index detected, and no NVIDIA card present")
-	}
-
-	return minor, nil
-}
-
-// Return string for minor number of nvidia device corresponding to the given pci id
-func findNvidiaMinor(pci string) (string, error) {
-	nvidiaPath := fmt.Sprintf("/proc/driver/nvidia/gpus/%s/information", pci)
-	buf, err := ioutil.ReadFile(nvidiaPath)
-	if err != nil {
-		return "", err
-	}
-
-	strBuf := strings.TrimSpace(string(buf))
-	idx := strings.Index(strBuf, "Device Minor:")
-	if idx != -1 {
-		idx += len("Device Minor:")
-		strBuf = strBuf[idx:]
-		strBuf = strings.TrimSpace(strBuf)
-		parts := strings.SplitN(strBuf, "\n", 2)
-		_, err = strconv.Atoi(parts[0])
-		if err == nil {
-			return parts[0], nil
-		}
-	}
-
-	minor, err := findNvidiaMinorOld()
-	if err == nil {
-		return minor, nil
-	}
-
-	return "", err
-}
-
-func deviceWantsAllGPUs(m map[string]string) bool {
-	return m["vendorid"] == "" && m["productid"] == "" && m["id"] == "" && m["pci"] == ""
-}
-
-func deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
-	const DRM_PATH = "/sys/class/drm/"
-	var gpus []gpuDevice
-	var nvidiaDevices []nvidiaGpuDevice
-	var cards []cardIds
-
-	// Load NVIDIA information (if available)
-	var nvidiaContainer *nvidiaContainerInfo
-
-	_, err := exec.LookPath("nvidia-container-cli")
-	if err == nil {
-		out, err := shared.RunCommand("nvidia-container-cli", "info", "--csv")
-		if err == nil {
-			r := csv.NewReader(strings.NewReader(out))
-			r.FieldsPerRecord = -1
-
-			nvidiaContainer = &nvidiaContainerInfo{}
-			nvidiaContainer.Cards = map[string]*nvidiaContainerCardInfo{}
-			line := 0
-			for {
-				record, err := r.Read()
-				if err == io.EOF {
-					break
-				}
-				line += 1
-
-				if err != nil {
-					continue
-				}
-
-				if line == 2 && len(record) >= 2 {
-					nvidiaContainer.NVRMVersion = record[0]
-					nvidiaContainer.CUDAVersion = record[1]
-				} else if line >= 4 {
-					nvidiaContainer.Cards[record[5]] = &nvidiaContainerCardInfo{
-						DeviceIndex:  record[0],
-						DeviceMinor:  record[1],
-						Model:        record[2],
-						Brand:        record[3],
-						UUID:         record[4],
-						PCIAddress:   record[5],
-						Architecture: record[6],
-					}
-				}
-			}
-		}
-	}
-
-	// Load PCI database
-	pciDB, err := pcidb.New()
-	if err != nil {
-		pciDB = nil
-	}
-
-	// Get the list of DRM devices
-	ents, err := ioutil.ReadDir(DRM_PATH)
-	if err != nil {
-		// No GPUs
-		if os.IsNotExist(err) {
-			return nil, nil, nil
-		}
-
-		return nil, nil, err
-	}
-
-	// Get the list of cards
-	devices := []string{}
-	for _, ent := range ents {
-		dev, err := filepath.EvalSymlinks(fmt.Sprintf("%s/%s/device", DRM_PATH, ent.Name()))
-		if err != nil {
-			continue
-		}
-
-		if !shared.StringInSlice(dev, devices) {
-			devices = append(devices, dev)
-		}
-	}
-
-	isNvidia := false
-	for _, device := range devices {
-		// The pci address == the name of the directory. So let's use
-		// this cheap way of retrieving it.
-		pciAddr := filepath.Base(device)
-
-		// Make sure that we are dealing with a GPU by looking whether
-		// the "drm" subfolder exists.
-		drm := filepath.Join(device, "drm")
-		drmEnts, err := ioutil.ReadDir(drm)
-		if err != nil {
-			if os.IsNotExist(err) {
-				continue
-			}
-		}
-
-		// Retrieve vendor ID.
-		vendorIdPath := filepath.Join(device, "vendor")
-		vendorId, err := ioutil.ReadFile(vendorIdPath)
-		if err != nil {
-			if os.IsNotExist(err) {
-				continue
-			}
-		}
-
-		// Retrieve device ID.
-		productIdPath := filepath.Join(device, "device")
-		productId, err := ioutil.ReadFile(productIdPath)
-		if err != nil {
-			if os.IsNotExist(err) {
-				continue
-			}
-		}
-
-		// Retrieve node ID
-		numaPath := fmt.Sprintf(filepath.Join(device, "numa_node"))
-		numaNode := uint64(0)
-		if shared.PathExists(numaPath) {
-			numaID, err := shared.ParseNumberFromFile(numaPath)
-			if err != nil {
-				continue
-			}
-
-			if numaID > 0 {
-				numaNode = uint64(numaID)
-			}
-		}
-
-		// Retrieve driver
-		driver := ""
-		driverVersion := ""
-		driverPath := filepath.Join(device, "driver")
-		if shared.PathExists(driverPath) {
-			target, err := os.Readlink(driverPath)
-			if err != nil {
-				continue
-			}
-
-			driver = filepath.Base(target)
-
-			out, err := ioutil.ReadFile(filepath.Join(driverPath, "module", "version"))
-			if err == nil {
-				driverVersion = strings.TrimSpace(string(out))
-			} else {
-				uname, err := shared.Uname()
-				if err != nil {
-					continue
-				}
-				driverVersion = uname.Release
-			}
-		}
-
-		// Store all associated subdevices, e.g. controlD64, renderD128.
-		// The name of the directory == the last part of the
-		// /dev/dri/controlD64 path. So drmEnt.Name() will give us
-		// controlD64.
-		for _, drmEnt := range drmEnts {
-			vendorTmp := strings.TrimSpace(string(vendorId))
-			productTmp := strings.TrimSpace(string(productId))
-			vendorTmp = strings.TrimPrefix(vendorTmp, "0x")
-			productTmp = strings.TrimPrefix(productTmp, "0x")
-			tmpGpu := gpuDevice{
-				pci:           pciAddr,
-				vendorID:      vendorTmp,
-				productID:     productTmp,
-				numaNode:      numaNode,
-				driver:        driver,
-				driverVersion: driverVersion,
-				path:          filepath.Join("/dev/dri", drmEnt.Name()),
-			}
-
-			// Fill vendor and product names
-			if pciDB != nil {
-				vendor, ok := pciDB.Vendors[tmpGpu.vendorID]
-				if ok {
-					tmpGpu.vendorName = vendor.Name
-
-					for _, product := range vendor.Products {
-						if product.ID == tmpGpu.productID {
-							tmpGpu.productName = product.Name
-							break
-						}
-					}
-				}
-			}
-
-			majMinPath := filepath.Join(drm, drmEnt.Name(), "dev")
-			majMinByte, err := ioutil.ReadFile(majMinPath)
-			if err != nil {
-				if os.IsNotExist(err) {
-					continue
-				}
-			}
-
-			majMin := strings.TrimSpace(string(majMinByte))
-			majMinSlice := strings.Split(string(majMin), ":")
-			if len(majMinSlice) != 2 {
-				continue
-			}
-
-			majorInt, err := strconv.Atoi(majMinSlice[0])
-			if err != nil {
-				continue
-			}
-
-			minorInt, err := strconv.Atoi(majMinSlice[1])
-			if err != nil {
-				continue
-			}
-
-			tmpGpu.major = majorInt
-			tmpGpu.minor = minorInt
-
-			isCard, err := regexp.MatchString("^card[0-9]+", drmEnt.Name())
-			if err != nil {
-				continue
-			}
-
-			// Find matching /dev/nvidia* entry for /dev/dri/card*
-			if tmpGpu.isNvidiaGpu() && isCard {
-				if !isNvidia {
-					isNvidia = true
-				}
-				tmpGpu.isNvidia = true
-
-				if !all {
-					minor, err := findNvidiaMinor(tmpGpu.pci)
-					if err == nil {
-						nvidiaPath := "/dev/nvidia" + minor
-						stat := unix.Stat_t{}
-						err = unix.Stat(nvidiaPath, &stat)
-						if err != nil {
-							if os.IsNotExist(err) {
-								continue
-							}
-
-							return nil, nil, err
-						}
-
-						tmpGpu.nvidia.path = nvidiaPath
-						tmpGpu.nvidia.major = shared.Major(stat.Rdev)
-						tmpGpu.nvidia.minor = shared.Minor(stat.Rdev)
-						tmpGpu.nvidia.id = strconv.Itoa(tmpGpu.nvidia.minor)
-
-						if nvidiaContainer != nil {
-							tmpGpu.nvidia.nvrmVersion = nvidiaContainer.NVRMVersion
-							tmpGpu.nvidia.cudaVersion = nvidiaContainer.CUDAVersion
-							nvidiaInfo, ok := nvidiaContainer.Cards[tmpGpu.pci]
-							if !ok {
-								nvidiaInfo, ok = nvidiaContainer.Cards[fmt.Sprintf("0000%v", tmpGpu.pci)]
-							}
-							if ok {
-								tmpGpu.nvidia.brand = nvidiaInfo.Brand
-								tmpGpu.nvidia.model = nvidiaInfo.Model
-								tmpGpu.nvidia.uuid = nvidiaInfo.UUID
-								tmpGpu.nvidia.architecture = nvidiaInfo.Architecture
-							}
-						}
-					}
-				}
-			}
-
-			if isCard {
-				// If it is a card it's minor number will be its id.
-				tmpGpu.id = strconv.Itoa(minorInt)
-				tmp := cardIds{
-					id:  tmpGpu.id,
-					pci: tmpGpu.pci,
-				}
-
-				cards = append(cards, tmp)
-			}
-
-			gpus = append(gpus, tmpGpu)
-		}
-	}
-
-	// We detected a Nvidia card, so let's collect all other nvidia devices
-	// that are not /dev/nvidia[0-9]+.
-	if isNvidia {
-		nvidiaEnts, err := ioutil.ReadDir("/dev")
-		if err != nil {
-			if os.IsNotExist(err) {
-				return nil, nil, err
-			}
-		}
-
-		validNvidia, err := regexp.Compile(`^nvidia[^0-9]+`)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		for _, nvidiaEnt := range nvidiaEnts {
-			if all {
-				if !strings.HasPrefix(nvidiaEnt.Name(), "nvidia") {
-					continue
-				}
-			} else {
-				if !validNvidia.MatchString(nvidiaEnt.Name()) {
-					continue
-				}
-			}
-
-			nvidiaPath := filepath.Join("/dev", nvidiaEnt.Name())
-			stat := unix.Stat_t{}
-			err = unix.Stat(nvidiaPath, &stat)
-			if err != nil {
-				continue
-			}
-
-			tmpNividiaGpu := nvidiaGpuDevice{
-				isCard: !validNvidia.MatchString(nvidiaEnt.Name()),
-				path:   nvidiaPath,
-				major:  shared.Major(stat.Rdev),
-				minor:  shared.Minor(stat.Rdev),
-			}
-
-			nvidiaDevices = append(nvidiaDevices, tmpNividiaGpu)
-		}
-	}
-
-	// Since we'll give users to ability to specify and id we need to group
-	// devices on the same PCI that belong to the same card by id.
-	for _, card := range cards {
-		for i := 0; i < len(gpus); i++ {
-			if gpus[i].pci == card.pci {
-				gpus[i].id = card.id
-			}
-		}
-	}
-
-	return gpus, nvidiaDevices, nil
-}
-
 func createUSBDevice(action string, vendor string, product string, major string, minor string, busnum string, devnum string, devname string, ueventParts []string, ueventLen int) (usbDevice, error) {
 	majorInt, err := strconv.Atoi(major)
 	if err != nil {

From b93797559c061b89fc9f7b6be3834d526cf88986 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:31:51 +0100
Subject: [PATCH 2/9] device/gpu: Adds gpu implementation

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/device/gpu.go | 184 +++++++++++++++++++++++++++++++---------------
 1 file changed, 125 insertions(+), 59 deletions(-)

diff --git a/lxd/device/gpu.go b/lxd/device/gpu.go
index d176930542..a984196e32 100644
--- a/lxd/device/gpu.go
+++ b/lxd/device/gpu.go
@@ -1,7 +1,19 @@
 package device
 
 import (
+	"encoding/csv"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/jaypipes/pcidb"
+	"golang.org/x/sys/unix"
 
 	"github.com/lxc/lxd/lxd/device/config"
 	"github.com/lxc/lxd/lxd/instance"
@@ -41,6 +53,10 @@ type gpuDevice struct {
 	nvidia   nvidiaGpuCard
 }
 
+func (g *gpuDevice) isNvidiaGpu() bool {
+	return strings.EqualFold(g.vendorID, "10de")
+}
+
 // /dev/nvidia[0-9]+
 type nvidiaGpuCard struct {
 	path  string
@@ -64,6 +80,28 @@ type nvidiaGpuDevice struct {
 	minor  int
 }
 
+// Nvidia container info
+type nvidiaContainerInfo struct {
+	Cards       map[string]*nvidiaContainerCardInfo
+	NVRMVersion string
+	CUDAVersion string
+}
+
+type nvidiaContainerCardInfo struct {
+	DeviceIndex  string
+	DeviceMinor  string
+	Model        string
+	Brand        string
+	UUID         string
+	PCIAddress   string
+	Architecture string
+}
+
+type cardIds struct {
+	id  string
+	pci string
+}
+
 // validateConfig checks the supplied config for correctness.
 func (d *gpu) validateConfig() error {
 	if d.instance.Type() != instance.TypeContainer {
@@ -112,6 +150,8 @@ func (d *gpu) Start() (*RunConfig, error) {
 		return nil, err
 	}
 
+	runConf := RunConfig{}
+
 	allGpus := d.deviceWantsAllGPUs(d.config)
 	gpus, nvidiaDevices, err := d.deviceLoadGpu(allGpus)
 	if err != nil {
@@ -129,11 +169,9 @@ func (d *gpu) Start() (*RunConfig, error) {
 		}
 
 		found = true
-
-		err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, false)
+		err := unixSetupCharDeviceNum(d.state, d.instance.DevicesPath(), fmt.Sprintf("unix.%s", d.name), d.config, gpu.major, gpu.minor, gpu.path, false, &runConf)
 		if err != nil {
-			logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
-			return err
+			return nil, err
 		}
 
 		if !gpu.isNvidia {
@@ -141,15 +179,12 @@ func (d *gpu) Start() (*RunConfig, error) {
 		}
 
 		if gpu.nvidia.path != "" {
-			err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path, false)
+			err = unixSetupCharDeviceNum(d.state, d.instance.DevicesPath(), fmt.Sprintf("unix.%s", d.name), d.config, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path, false, &runConf)
 			if err != nil {
-				logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
-				return err
+				return nil, err
 			}
 		} else if !allGpus {
-			errMsg := fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
-			logger.Errorf("%s", errMsg)
-			return errMsg
+			return nil, fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
 		}
 
 		sawNvidia = true
@@ -157,59 +192,36 @@ func (d *gpu) Start() (*RunConfig, error) {
 
 	if sawNvidia {
 		for _, gpu := range nvidiaDevices {
-			if shared.IsTrue(c.expandedConfig["nvidia.runtime"]) {
+			// tomp TODO figure out how to access container level config properties!
+			if shared.IsTrue(d.config["nvidia.runtime"]) {
 				if !gpu.isCard {
 					continue
 				}
 			}
 
-			if c.deviceExistsInDevicesFolder(k, gpu.path) {
+			prefix := fmt.Sprintf("unix.%s", d.name)
+
+			if UnixDeviceExistsInDevicesFolder(d.instance.DevicesPath(), prefix, gpu.path) {
 				continue
 			}
 
-			err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, false)
+			err = unixSetupCharDeviceNum(d.state, d.instance.DevicesPath(), prefix, d.config, gpu.major, gpu.minor, gpu.path, false, &runConf)
 			if err != nil {
-				logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
-				return err
+				return nil, err
 			}
 		}
 	}
 
 	if !found {
-		msg := "Failed to detect requested GPU device"
-		logger.Error(msg)
-		return fmt.Errorf(msg)
+		return nil, fmt.Errorf("Failed to detect requested GPU device")
 	}
 
-	return nil, nil
+	return &runConf, nil
 }
 
-func (d *gpu) configureUnixDeviceNum(name string, config config.Device, major int, minor int, path string, defaultMode bool, runConf *RunConfig) error {
-	paths, err := UnixCreateDevice(s, nil, devicesPath, devPrefix, dummyDevice, false)
-	if err != nil {
-		return err
-	}
-	devPath := paths[0]
-
-	// Instruct liblxc to perform the mount.
-	runConf.Mounts = append(runConf.Mounts, MountEntryItem{
-		DevPath:    devPath,
-		TargetPath: relDestPath,
-		FSType:     "none",
-		Opts:       []string{"bind", "create=file"},
-	})
-
-	// Add the new device cgroup rule.
-	dType, dMajor, dMinor, err := UnixGetDeviceAttributes(devPath)
-	if err != nil {
-		return err
-	}
-
-	// Instruct liblxc to setup the cgroup rule.
-	runConf.CGroups = append(runConf.CGroups, RunConfigItem{
-		Key:   "devices.allow",
-		Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor),
-	})
+// Stop is run when the device is removed from the instance.
+func (d *gpu) Stop() (*RunConfig, error) {
+	return nil, nil
 }
 
 // deviceWantsAllGPUs whether the LXD device wants to passthrough all GPUs on the host.
@@ -219,7 +231,7 @@ func (d *gpu) deviceWantsAllGPUs(m map[string]string) bool {
 
 // deviceLoadGpu probes the system for information about the availble GPUs.
 func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
-	const DRM_PATH = "/sys/class/drm/"
+	const drmPath = "/sys/class/drm/"
 	var gpus []gpuDevice
 	var nvidiaDevices []nvidiaGpuDevice
 	var cards []cardIds
@@ -242,7 +254,7 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
 				if err == io.EOF {
 					break
 				}
-				line += 1
+				line++
 
 				if err != nil {
 					continue
@@ -273,7 +285,7 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
 	}
 
 	// Get the list of DRM devices
-	ents, err := ioutil.ReadDir(DRM_PATH)
+	ents, err := ioutil.ReadDir(drmPath)
 	if err != nil {
 		// No GPUs
 		if os.IsNotExist(err) {
@@ -286,7 +298,7 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
 	// Get the list of cards
 	devices := []string{}
 	for _, ent := range ents {
-		dev, err := filepath.EvalSymlinks(fmt.Sprintf("%s/%s/device", DRM_PATH, ent.Name()))
+		dev, err := filepath.EvalSymlinks(fmt.Sprintf("%s/%s/device", drmPath, ent.Name()))
 		if err != nil {
 			continue
 		}
@@ -313,8 +325,8 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
 		}
 
 		// Retrieve vendor ID.
-		vendorIdPath := filepath.Join(device, "vendor")
-		vendorId, err := ioutil.ReadFile(vendorIdPath)
+		vendorIDPath := filepath.Join(device, "vendor")
+		vendorID, err := ioutil.ReadFile(vendorIDPath)
 		if err != nil {
 			if os.IsNotExist(err) {
 				continue
@@ -322,8 +334,8 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
 		}
 
 		// Retrieve device ID.
-		productIdPath := filepath.Join(device, "device")
-		productId, err := ioutil.ReadFile(productIdPath)
+		productIDPath := filepath.Join(device, "device")
+		productID, err := ioutil.ReadFile(productIDPath)
 		if err != nil {
 			if os.IsNotExist(err) {
 				continue
@@ -373,8 +385,8 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
 		// /dev/dri/controlD64 path. So drmEnt.Name() will give us
 		// controlD64.
 		for _, drmEnt := range drmEnts {
-			vendorTmp := strings.TrimSpace(string(vendorId))
-			productTmp := strings.TrimSpace(string(productId))
+			vendorTmp := strings.TrimSpace(string(vendorID))
+			productTmp := strings.TrimSpace(string(productID))
 			vendorTmp = strings.TrimPrefix(vendorTmp, "0x")
 			productTmp = strings.TrimPrefix(productTmp, "0x")
 			tmpGpu := gpuDevice{
@@ -442,7 +454,7 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
 				tmpGpu.isNvidia = true
 
 				if !all {
-					minor, err := findNvidiaMinor(tmpGpu.pci)
+					minor, err := d.findNvidiaMinor(tmpGpu.pci)
 					if err == nil {
 						nvidiaPath := "/dev/nvidia" + minor
 						stat := unix.Stat_t{}
@@ -550,7 +562,61 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
 	return gpus, nvidiaDevices, nil
 }
 
-// Stop is run when the device is removed from the instance.
-func (d *gpu) Stop() (*RunConfig, error) {
-	return nil, nil
+// findNvidiaMinorOld fallback for old drivers which don't provide "Device Minor:".
+func (d *gpu) findNvidiaMinorOld() (string, error) {
+	var minor string
+
+	// For now, just handle most common case (single nvidia card)
+	ents, err := ioutil.ReadDir("/dev")
+	if err != nil {
+		return "", err
+	}
+
+	rp := regexp.MustCompile("^nvidia([0-9]+)$")
+	for _, ent := range ents {
+		matches := rp.FindStringSubmatch(ent.Name())
+		if matches == nil {
+			continue
+		}
+
+		if minor != "" {
+			return "", fmt.Errorf("No device minor index detected, and more than one NVIDIA card present")
+		}
+		minor = matches[1]
+	}
+
+	if minor == "" {
+		return "", fmt.Errorf("No device minor index detected, and no NVIDIA card present")
+	}
+
+	return minor, nil
+}
+
+// findNvidiaMinor returns minor number of nvidia device corresponding to the given pci id.
+func (d *gpu) findNvidiaMinor(pci string) (string, error) {
+	nvidiaPath := fmt.Sprintf("/proc/driver/nvidia/gpus/%s/information", pci)
+	buf, err := ioutil.ReadFile(nvidiaPath)
+	if err != nil {
+		return "", err
+	}
+
+	strBuf := strings.TrimSpace(string(buf))
+	idx := strings.Index(strBuf, "Device Minor:")
+	if idx != -1 {
+		idx += len("Device Minor:")
+		strBuf = strBuf[idx:]
+		strBuf = strings.TrimSpace(strBuf)
+		parts := strings.SplitN(strBuf, "\n", 2)
+		_, err = strconv.Atoi(parts[0])
+		if err == nil {
+			return parts[0], nil
+		}
+	}
+
+	minor, err := d.findNvidiaMinorOld()
+	if err == nil {
+		return minor, nil
+	}
+
+	return "", err
 }

From 18967c925bb80caf1a010e5fc66e1a3b5b3e2fda Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:32:43 +0100
Subject: [PATCH 3/9] container: Removes gpu validation as moved to device
 package

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/container.go | 31 -------------------------------
 1 file changed, 31 deletions(-)

diff --git a/lxd/container.go b/lxd/container.go
index 10f9e39284..0e169e7fbc 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -194,25 +194,6 @@ func containerValidDeviceConfigKey(t, k string) bool {
 		default:
 			return false
 		}
-	case "gpu":
-		switch k {
-		case "vendorid":
-			return true
-		case "productid":
-			return true
-		case "id":
-			return true
-		case "pci":
-			return true
-		case "mode":
-			return true
-		case "gid":
-			return true
-		case "uid":
-			return true
-		default:
-			return false
-		}
 	case "none":
 		return false
 	default:
@@ -425,18 +406,6 @@ func containerValidDevices(state *state.State, cluster *db.Cluster, devices conf
 			}
 		} else if m["type"] == "usb" {
 			// Nothing needed for usb.
-		} else if m["type"] == "gpu" {
-			if m["pci"] != "" && !shared.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s", m["pci"])) {
-				return fmt.Errorf("Invalid PCI address (no device found): %s", m["pci"])
-			}
-
-			if m["pci"] != "" && (m["id"] != "" || m["productid"] != "" || m["vendorid"] != "") {
-				return fmt.Errorf("Cannot use id, productid or vendorid when pci is set")
-			}
-
-			if m["id"] != "" && (m["pci"] != "" || m["productid"] != "" || m["vendorid"] != "") {
-				return fmt.Errorf("Cannot use pci, productid or vendorid when id is set")
-			}
 		} else if m["type"] == "none" {
 			continue
 		} else {

From 01004261b58eb65ce24937389a2630e75765b9fe Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:35:26 +0100
Subject: [PATCH 4/9] gpu cont

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/device/gpu.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lxd/device/gpu.go b/lxd/device/gpu.go
index a984196e32..4b8fe1f696 100644
--- a/lxd/device/gpu.go
+++ b/lxd/device/gpu.go
@@ -201,7 +201,7 @@ func (d *gpu) Start() (*RunConfig, error) {
 
 			prefix := fmt.Sprintf("unix.%s", d.name)
 
-			if UnixDeviceExistsInDevicesFolder(d.instance.DevicesPath(), prefix, gpu.path) {
+			if UnixDeviceExists(d.instance.DevicesPath(), prefix, gpu.path) {
 				continue
 			}
 

From 3a665ac7f8be4ca37bf658eb04ffe4d57c301633 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:35:37 +0100
Subject: [PATCH 5/9] device: Links gpu device

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/device/device.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lxd/device/device.go b/lxd/device/device.go
index a7566dda42..5c6833e1e6 100644
--- a/lxd/device/device.go
+++ b/lxd/device/device.go
@@ -10,8 +10,9 @@ import (
 // devTypes defines supported top-level device type creation functions.
 var devTypes = map[string]func(config.Device) device{
 	"nic":        nicLoadByType,
-	"proxy":      func(c config.Device) device { return &proxy{} },
 	"infiniband": infinibandLoadByType,
+	"proxy":      func(c config.Device) device { return &proxy{} },
+	"gpu":        func(c config.Device) device { return &gpu{} },
 }
 
 // VolatileSetter is a function that accepts one or more key/value strings to save into the LXD

From ecea86cff27ffac96ed1f4e45f11750a53bbae2d Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:36:22 +0100
Subject: [PATCH 6/9] device/device/utils/infiniband: Updates use of unix
 functions

The unix device create functions have been updated to return info about the device created, which simplifies some of the infiniband setup.

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/device/device_utils_infiniband.go | 44 ++++-----------------------
 1 file changed, 6 insertions(+), 38 deletions(-)

diff --git a/lxd/device/device_utils_infiniband.go b/lxd/device/device_utils_infiniband.go
index 01d606854b..a05229cde3 100644
--- a/lxd/device/device_utils_infiniband.go
+++ b/lxd/device/device_utils_infiniband.go
@@ -319,11 +319,10 @@ func infinibandAddDevicesPerPort(s *state.State, devicesPath string, deviceName
 			break
 		}
 
-		paths, err := UnixCreateDevice(s, nil, devicesPath, devPrefix, dummyDevice, false)
+		d, err := UnixCreateDevice(s, nil, devicesPath, devPrefix, dummyDevice, false)
 		if err != nil {
 			return err
 		}
-		devPath := paths[0]
 
 		// If an existing device with the same path in the instance exists, then do not
 		// request it be mounted again.
@@ -333,22 +332,16 @@ func infinibandAddDevicesPerPort(s *state.State, devicesPath string, deviceName
 
 		// Instruct liblxc to perform the mount.
 		runConf.Mounts = append(runConf.Mounts, MountEntryItem{
-			DevPath:    devPath,
-			TargetPath: relDestPath,
+			DevPath:    d.HostPath,
+			TargetPath: d.RelativePath,
 			FSType:     "none",
 			Opts:       []string{"bind", "create=file"},
 		})
 
-		// Add the new device cgroup rule.
-		dType, dMajor, dMinor, err := UnixGetDeviceAttributes(devPath)
-		if err != nil {
-			return err
-		}
-
 		// Instruct liblxc to setup the cgroup rule.
 		runConf.CGroups = append(runConf.CGroups, RunConfigItem{
 			Key:   "devices.allow",
-			Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor),
+			Value: fmt.Sprintf("%s %d:%d rwm", d.Type, d.Major, d.Minor),
 		})
 	}
 
@@ -359,39 +352,14 @@ func infinibandAddDevicesPerFun(s *state.State, devicesPath string, deviceName s
 	for _, unixCharDev := range ifDev.PerFunDevices {
 		destPath := fmt.Sprintf("/dev/infiniband/%s", unixCharDev)
 		uniqueDevPrefix := fmt.Sprintf("%s.%s", IBDevPrefix, deviceName)
-		relativeDestPath := fmt.Sprintf("dev/infiniband/%s", unixCharDev)
-		uniqueDevName := fmt.Sprintf("%s.%s", uniqueDevPrefix, strings.Replace(relativeDestPath, "/", "-", -1))
-		hostDevPath := filepath.Join(devicesPath, uniqueDevName)
-
 		dummyDevice := config.Device{
 			"source": destPath,
 		}
 
-		// Instruct liblxc to perform the mount.
-		runConf.Mounts = append(runConf.Mounts, MountEntryItem{
-			DevPath:    hostDevPath,
-			TargetPath: relativeDestPath,
-			FSType:     "none",
-			Opts:       []string{"bind", "create=file"},
-		})
-
-		paths, err := UnixCreateDevice(s, nil, devicesPath, uniqueDevPrefix, dummyDevice, false)
-		if err != nil {
-			return err
-		}
-		devPath := paths[0]
-
-		// Add the new device cgroup rule.
-		dType, dMajor, dMinor, err := UnixGetDeviceAttributes(devPath)
+		err := unixSetupDevice(s, devicesPath, uniqueDevPrefix, dummyDevice, false, runConf)
 		if err != nil {
 			return err
 		}
-
-		// Instruct liblxc to setup the cgroup rule.
-		runConf.CGroups = append(runConf.CGroups, RunConfigItem{
-			Key:   "devices.allow",
-			Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor),
-		})
 	}
 
 	return nil
@@ -483,7 +451,7 @@ func infinibandRemoveDevices(devicesPath string, deviceName string, runConf *Run
 			"source": absPath,
 		}
 
-		dType, dMajor, dMinor, err := instanceUnixGetDeviceAttributes(devicesPath, ourPrefix, dummyDevice)
+		dType, dMajor, dMinor, err := unixGetInstanceDeviceAttributes(devicesPath, ourPrefix, dummyDevice)
 		if err != nil {
 			return err
 		}

From 59b807c9e08b29f8d1f9b03d8b1e1ff359d2f3e3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:38:18 +0100
Subject: [PATCH 7/9] device/device/utils/unix: Renames
 instanceUnixGetDeviceAttributes to unixGetInstanceDeviceAttributes

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/device/device_utils_unix.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lxd/device/device_utils_unix.go b/lxd/device/device_utils_unix.go
index 45c816494a..355937d198 100644
--- a/lxd/device/device_utils_unix.go
+++ b/lxd/device/device_utils_unix.go
@@ -16,10 +16,10 @@ import (
 	"github.com/lxc/lxd/shared/logger"
 )
 
-// instanceUnixGetDeviceAttributes returns the UNIX device attributes for an instance device.
+// unixGetInstanceDeviceAttributes returns the UNIX device attributes for an instance device.
 // Uses supplied device config for device properties, and if they haven't been set, falls back to
 // using UnixGetDeviceAttributes() to directly query an existing device file.
-func instanceUnixGetDeviceAttributes(devicesPath string, prefix string, config config.Device) (string, int, int, error) {
+func unixGetInstanceDeviceAttributes(devicesPath string, prefix string, config config.Device) (string, int, int, error) {
 	// Check if we've been passed major and minor numbers already.
 	var tmp int
 	var err error

From 400dc4abea33713e4247b1dba2217f6924d74a81 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:39:31 +0100
Subject: [PATCH 8/9] device/device/utils/unix: Improves device creation
 functions

Updates UnixCreateDevice() to return a UnixDevice struct with all the info about the created device.
This is to avoid having to have duplicate code elsewhere to stat the device to find out about it's properties.

Also adds unixSetupDevice() and unixSetupCharDeviceNum() helper functions that will call UnixCreateDevice() and then use the returned information to configure the RunConfig struct to instruct LXD to attach the device to the instance.

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/device/device_utils_unix.go | 143 +++++++++++++++++++++++++-------
 1 file changed, 112 insertions(+), 31 deletions(-)

diff --git a/lxd/device/device_utils_unix.go b/lxd/device/device_utils_unix.go
index 355937d198..605e197746 100644
--- a/lxd/device/device_utils_unix.go
+++ b/lxd/device/device_utils_unix.go
@@ -105,12 +105,31 @@ func unixGetDeviceModeOct(strmode string) (int, error) {
 	return int(i), nil
 }
 
-// UnixCreateDevice Unix devices handling.
-func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath string, prefix string, m config.Device, defaultMode bool) ([]string, error) {
+// UnixDevice contains information about a created UNIX device.
+type UnixDevice struct {
+	HostPath     string      // Absolute path to the device on the host.
+	RelativePath string      // Relative path where the device will be mounted inside instance.
+	Type         string      // Type of device; c (for char) or b for (block).
+	Major        int         // Major number.
+	Minor        int         // Minor number.
+	Mode         os.FileMode // File mode.
+	UID          int         // Owner UID.
+	GID          int         // Owner GID.
+}
+
+// UnixCreateDevice creates a UNIX device (either block or char). If the supplied device config map
+// contains a major and minor number for the device, then a stat is avoided, otherwise this info
+// retrieved from the origin device. Similarly, if a mode is supplied in the device config map or
+// defaultMode is set as true, then the device is created with the supplied or default mode (0660)
+// respectively, otherwise the origin device's mode is used. If the device config doesn't contain
+// a type field then it defaults to created a unix-char device. The ownership of the created device
+// defaults to root (0) but can be specified with the uid and gid fields in the device config map.
+// It returns a UnixDevice containing information about the device created.
+func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath string, prefix string, m config.Device, defaultMode bool) (*UnixDevice, error) {
 	var err error
-	var major, minor int
+	d := UnixDevice{}
 
-	// Extra checks for nesting
+	// Extra checks for nesting.
 	if s.OS.RunningInUserNS {
 		for key, value := range m {
 			if shared.StringInSlice(key, []string{"major", "minor", "mode", "uid", "gid"}) && value != "" {
@@ -125,71 +144,70 @@ func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath stri
 	}
 	srcPath = shared.HostPath(srcPath)
 
-	// Get the major/minor of the device we want to create
+	// Get the major/minor of the device we want to create.
 	if m["major"] == "" && m["minor"] == "" {
-		// If no major and minor are set, use those from the device on the host
-		_, major, minor, err = UnixGetDeviceAttributes(srcPath)
+		// If no major and minor are set, use those from the device on the host.
+		_, d.Major, d.Minor, err = UnixGetDeviceAttributes(srcPath)
 		if err != nil {
 			return nil, fmt.Errorf("Failed to get device attributes for %s: %s", m["path"], err)
 		}
 	} else if m["major"] == "" || m["minor"] == "" {
 		return nil, fmt.Errorf("Both major and minor must be supplied for device: %s", m["path"])
 	} else {
-		major, err = strconv.Atoi(m["major"])
+		d.Major, err = strconv.Atoi(m["major"])
 		if err != nil {
 			return nil, fmt.Errorf("Bad major %s in device %s", m["major"], m["path"])
 		}
 
-		minor, err = strconv.Atoi(m["minor"])
+		d.Minor, err = strconv.Atoi(m["minor"])
 		if err != nil {
 			return nil, fmt.Errorf("Bad minor %s in device %s", m["minor"], m["path"])
 		}
 	}
 
-	// Get the device mode
-	mode := os.FileMode(0660)
+	// Get the device mode (defaults to 0660 if not supplied).
+	d.Mode = os.FileMode(0660)
 	if m["mode"] != "" {
 		tmp, err := unixGetDeviceModeOct(m["mode"])
 		if err != nil {
 			return nil, fmt.Errorf("Bad mode %s in device %s", m["mode"], m["path"])
 		}
-		mode = os.FileMode(tmp)
+		d.Mode = os.FileMode(tmp)
 	} else if !defaultMode {
-		mode, err = shared.GetPathMode(srcPath)
+		d.Mode, err = shared.GetPathMode(srcPath)
 		if err != nil {
 			errno, isErrno := shared.GetErrno(err)
 			if !isErrno || errno != unix.ENOENT {
 				return nil, fmt.Errorf("Failed to retrieve mode of device %s: %s", m["path"], err)
 			}
-			mode = os.FileMode(0660)
+			d.Mode = os.FileMode(0660)
 		}
 	}
 
 	if m["type"] == "unix-block" {
-		mode |= unix.S_IFBLK
+		d.Mode |= unix.S_IFBLK
+		d.Type = "b"
 	} else {
-		mode |= unix.S_IFCHR
+		d.Mode |= unix.S_IFCHR
+		d.Type = "c"
 	}
 
-	// Get the device owner
-	uid := 0
-	gid := 0
-
+	// Get the device owner.
 	if m["uid"] != "" {
-		uid, err = strconv.Atoi(m["uid"])
+		d.UID, err = strconv.Atoi(m["uid"])
 		if err != nil {
 			return nil, fmt.Errorf("Invalid uid %s in device %s", m["uid"], m["path"])
 		}
 	}
 
 	if m["gid"] != "" {
-		gid, err = strconv.Atoi(m["gid"])
+		d.GID, err = strconv.Atoi(m["gid"])
 		if err != nil {
 			return nil, fmt.Errorf("Invalid gid %s in device %s", m["gid"], m["path"])
 		}
 	}
 
-	// Create the devices directory if missing
+	// Create the devices directory if missing.
 	if !shared.PathExists(devicesPath) {
 		os.Mkdir(devicesPath, 0711)
 		if err != nil {
@@ -205,21 +223,24 @@ func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath stri
 	devName := fmt.Sprintf("%s.%s", strings.Replace(prefix, "/", "-", -1), strings.Replace(relativeDestPath, "/", "-", -1))
 	devPath := filepath.Join(devicesPath, devName)
 
-	// Create the new entry
+	// Create the new entry.
 	if !s.OS.RunningInUserNS {
-		encodedDeviceNumber := (minor & 0xff) | (major << 8) | ((minor & ^0xff) << 12)
-		err := unix.Mknod(devPath, uint32(mode), encodedDeviceNumber)
+		// tomp TODO check this implementation is OK with stgraber.
+		encodedDeviceNumber := (d.Minor & 0xff) | (d.Major << 8) | ((d.Minor & ^0xff) << 12)
+		devNum := int(unix.Mkdev(uint32(d.Major), uint32(d.Minor)))
+		logger.Errorf("tomp mknod devNum: old: %v new: %v", encodedDeviceNumber, devNum)
+		err := unix.Mknod(devPath, uint32(d.Mode), devNum)
 		if err != nil {
 			return nil, fmt.Errorf("Failed to create device %s for %s: %s", devPath, m["path"], err)
 		}
 
-		err = os.Chown(devPath, uid, gid)
+		err = os.Chown(devPath, d.UID, d.GID)
 		if err != nil {
 			return nil, fmt.Errorf("Failed to chown device %s: %s", devPath, err)
 		}
 
-		// Needed as mknod respects the umask
-		err = os.Chmod(devPath, mode)
+		// Needed as mknod respects the umask.
+		err = os.Chmod(devPath, d.Mode)
 		if err != nil {
 			return nil, fmt.Errorf("Failed to chmod device %s: %s", devPath, err)
 		}
@@ -227,7 +248,7 @@ func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath stri
 		if idmapSet != nil {
 			err := idmapSet.ShiftFile(devPath)
 			if err != nil {
-				// uidshift failing is weird, but not a big problem.  Log and proceed
+				// uidshift failing is weird, but not a big problem. Log and proceed.
 				logger.Debugf("Failed to uidshift device %s: %s\n", m["path"], err)
 			}
 		}
@@ -244,5 +265,65 @@ func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath stri
 		}
 	}
 
-	return []string{devPath, relativeDestPath}, nil
+	d.HostPath = devPath
+	d.RelativePath = relativeDestPath
+	return &d, nil
+}
+
+// unixSetupDevice creates a UNIX device on host and then configures supplied RunConfig with the
+// mount and cgroup rule instructions to have it be attached to the instance. If defaultMode is true
+// or mode is supplied in the device config then the origin device does not need to be accessed for
+// its file mode.
+func unixSetupDevice(s *state.State, devicesPath string, prefix string, m config.Device, defaultMode bool, runConf *RunConfig) error {
+	// Create the device on the host.
+	d, err := UnixCreateDevice(s, nil, devicesPath, prefix, m, defaultMode)
+	if err != nil {
+		return fmt.Errorf("Failed to setup device: %s", err)
+	}
+
+	// Instruct liblxc to perform the mount.
+	runConf.Mounts = append(runConf.Mounts, MountEntryItem{
+		DevPath:    d.HostPath,
+		TargetPath: d.RelativePath,
+		FSType:     "none",
+		Opts:       []string{"bind", "create=file"},
+	})
+
+	// Instruct liblxc to setup the cgroup rule.
+	runConf.CGroups = append(runConf.CGroups, RunConfigItem{
+		Key:   "devices.allow",
+		Value: fmt.Sprintf("%s %d:%d rwm", d.Type, d.Major, d.Minor),
+	})
+
+	return nil
+}
+
+// unixSetupCharDeviceNum calls unixSetupDevice and overrides the supplied device config with the
+// type as "unix-char" and the supplied major and minor numbers. This function can be used when you
+// already know the device's major and minor numbers to avoid unixSetupDevice() having to stat the
+// device to ascertain these attributes. If defaultMode is true or mode is supplied in the device
+// config then the origin device does not need to be accessed for its file mode.
+func unixSetupCharDeviceNum(s *state.State, devicesPath string, prefix string, m config.Device, major int, minor int, path string, defaultMode bool, runConf *RunConfig) error {
+	configCopy := config.Device{}
+	for k, v := range m {
+		configCopy[k] = v
+	}
+
+	// Overridng these in the config copy should avoid the need for unixSetupDevice to stat
+	// the origin device to ascertain this information.
+	configCopy["type"] = "unix-char"
+	configCopy["major"] = fmt.Sprintf("%d", major)
+	configCopy["minor"] = fmt.Sprintf("%d", minor)
+	configCopy["path"] = path
+
+	return unixSetupDevice(s, devicesPath, prefix, configCopy, defaultMode, runConf)
+}
+
+// UnixDeviceExists checks if the unix device already exists in devices path.
+func UnixDeviceExists(devicesPath string, prefix string, path string) bool {
+	relativeDestPath := strings.TrimPrefix(path, "/")
+	devName := fmt.Sprintf("%s.%s", strings.Replace(prefix, "/", "-", -1), strings.Replace(relativeDestPath, "/", "-", -1))
+	devPath := filepath.Join(devicesPath, devName)
+
+	return shared.PathExists(devPath)
 }

From 4f506751bad5a7cb8e53337a7ef8ebd04a7bca50 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:43:39 +0100
Subject: [PATCH 9/9] container/lxc: Updates gpu device support to use device
 package

Also updates use of device.UnixCreateDevice() to use the new UnixDevice struct returned.

Removes gpu related code that has moved into device package.

Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
 lxd/container_lxc.go | 230 +++----------------------------------------
 1 file changed, 12 insertions(+), 218 deletions(-)

diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 14948f7244..ce86f19a56 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -2283,7 +2283,7 @@ func (c *containerLXC) setupUnixDevice(prefix string, dev config.Device, major i
 		return err
 	}
 
-	paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, temp, defaultMode)
+	d, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, temp, defaultMode)
 	if err != nil {
 		logger.Debug("Failed to create device", log.Ctx{"err": err, "device": prefix})
 		if createMustSucceed {
@@ -2293,8 +2293,8 @@ func (c *containerLXC) setupUnixDevice(prefix string, dev config.Device, major i
 		return nil
 	}
 
-	devPath := shared.EscapePathFstab(paths[0])
-	tgtPath := shared.EscapePathFstab(paths[1])
+	devPath := shared.EscapePathFstab(d.HostPath)
+	tgtPath := shared.EscapePathFstab(d.RelativePath)
 	val := fmt.Sprintf("%s %s none bind,create=file 0 0", devPath, tgtPath)
 
 	return lxcSetConfigItem(c.c, "lxc.mount.entry", val)
@@ -2535,7 +2535,7 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
 			}
 
 			// Unix device
-			paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), fmt.Sprintf("unix.%s", k), m, true)
+			d, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), fmt.Sprintf("unix.%s", k), m, true)
 			if err != nil {
 				// Deal with device hotplug
 				if m["required"] == "" || shared.IsTrue(m["required"]) {
@@ -2555,7 +2555,7 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
 				}
 				continue
 			}
-			devPath := paths[0]
+			devPath := d.HostPath
 			if c.isCurrentlyPrivileged() && !c.state.OS.RunningInUserNS && c.state.OS.CGroupDevicesController {
 				// Add the new device cgroup rule
 				dType, dMajor, dMinor, err := device.UnixGetDeviceAttributes(devPath)
@@ -2588,67 +2588,6 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
 					return "", postStartHooks, err
 				}
 			}
-		} else if m["type"] == "gpu" {
-			allGpus := deviceWantsAllGPUs(m)
-			gpus, nvidiaDevices, err := deviceLoadGpu(allGpus)
-			if err != nil {
-				return "", postStartHooks, err
-			}
-
-			sawNvidia := false
-			found := false
-			for _, gpu := range gpus {
-				if (m["vendorid"] != "" && gpu.vendorID != m["vendorid"]) ||
-					(m["pci"] != "" && gpu.pci != m["pci"]) ||
-					(m["productid"] != "" && gpu.productID != m["productid"]) ||
-					(m["id"] != "" && gpu.id != m["id"]) {
-					continue
-				}
-
-				found = true
-
-				err := c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, true, false)
-				if err != nil {
-					return "", postStartHooks, err
-				}
-
-				if !gpu.isNvidia {
-					continue
-				}
-
-				if gpu.nvidia.path != "" {
-					err = c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path, true, false)
-					if err != nil {
-						return "", postStartHooks, err
-					}
-				} else if !allGpus {
-					errMsg := fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
-					logger.Errorf("%s", errMsg)
-					return "", postStartHooks, errMsg
-				}
-
-				sawNvidia = true
-			}
-
-			if sawNvidia {
-				for _, gpu := range nvidiaDevices {
-					if shared.IsTrue(c.expandedConfig["nvidia.runtime"]) {
-						if !gpu.isCard {
-							continue
-						}
-					}
-					err := c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, true, false)
-					if err != nil {
-						return "", postStartHooks, err
-					}
-				}
-			}
-
-			if !found {
-				msg := "Failed to detect requested GPU device"
-				logger.Error(msg)
-				return "", postStartHooks, fmt.Errorf(msg)
-			}
 		} else if m["type"] == "disk" {
 			if m["path"] != "/" {
 				diskDevices[k] = m
@@ -4989,7 +4928,7 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
 					destPath = m["source"]
 				}
 
-				if !c.deviceExistsInDevicesFolder(prefix, destPath) && (m["required"] != "" && !shared.IsTrue(m["required"])) {
+				if !device.UnixDeviceExists(c.DevicesPath(), prefix, destPath) && (m["required"] != "" && !shared.IsTrue(m["required"])) {
 					continue
 				}
 
@@ -5021,73 +4960,6 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
 						return err
 					}
 				}
-			} else if m["type"] == "gpu" {
-				allGpus := deviceWantsAllGPUs(m)
-				gpus, nvidiaDevices, err := deviceLoadGpu(allGpus)
-				if err != nil {
-					return err
-				}
-
-				for _, gpu := range gpus {
-					if (m["vendorid"] != "" && gpu.vendorID != m["vendorid"]) ||
-						(m["pci"] != "" && gpu.pci != m["pci"]) ||
-						(m["productid"] != "" && gpu.productID != m["productid"]) ||
-						(m["id"] != "" && gpu.id != m["id"]) {
-						continue
-					}
-
-					err := c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path)
-					if err != nil {
-						logger.Error("Failed to remove GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
-						return err
-					}
-
-					if !gpu.isNvidia {
-						continue
-					}
-
-					if gpu.nvidia.path != "" {
-						err = c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path)
-						if err != nil {
-							logger.Error("Failed to remove GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
-							return err
-						}
-					} else if !allGpus {
-						errMsg := fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
-						logger.Errorf("%s", errMsg)
-						return errMsg
-					}
-				}
-
-				nvidiaExists := false
-				for _, gpu := range gpus {
-					if gpu.nvidia.path != "" {
-						if c.deviceExistsInDevicesFolder(fmt.Sprintf("unix.%s", k), gpu.path) {
-							nvidiaExists = true
-							break
-						}
-					}
-				}
-
-				if !nvidiaExists {
-					for _, gpu := range nvidiaDevices {
-						if shared.IsTrue(c.expandedConfig["nvidia.runtime"]) {
-							if !gpu.isCard {
-								continue
-							}
-						}
-
-						if !c.deviceExistsInDevicesFolder(fmt.Sprintf("unix.%s", k), gpu.path) {
-							continue
-						}
-
-						err = c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path)
-						if err != nil {
-							logger.Error("Failed to remove GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
-							return err
-						}
-					}
-				}
 			}
 		}
 
@@ -5120,75 +4992,6 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
 						logger.Error("Failed to insert usb device", log.Ctx{"err": err, "usb": usb, "container": c.Name()})
 					}
 				}
-			} else if m["type"] == "gpu" {
-				allGpus := deviceWantsAllGPUs(m)
-				gpus, nvidiaDevices, err := deviceLoadGpu(allGpus)
-				if err != nil {
-					return err
-				}
-
-				sawNvidia := false
-				found := false
-				for _, gpu := range gpus {
-					if (m["vendorid"] != "" && gpu.vendorID != m["vendorid"]) ||
-						(m["pci"] != "" && gpu.pci != m["pci"]) ||
-						(m["productid"] != "" && gpu.productID != m["productid"]) ||
-						(m["id"] != "" && gpu.id != m["id"]) {
-						continue
-					}
-
-					found = true
-
-					err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, false)
-					if err != nil {
-						logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
-						return err
-					}
-
-					if !gpu.isNvidia {
-						continue
-					}
-
-					if gpu.nvidia.path != "" {
-						err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path, false)
-						if err != nil {
-							logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
-							return err
-						}
-					} else if !allGpus {
-						errMsg := fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
-						logger.Errorf("%s", errMsg)
-						return errMsg
-					}
-
-					sawNvidia = true
-				}
-
-				if sawNvidia {
-					for _, gpu := range nvidiaDevices {
-						if shared.IsTrue(c.expandedConfig["nvidia.runtime"]) {
-							if !gpu.isCard {
-								continue
-							}
-						}
-
-						if c.deviceExistsInDevicesFolder(k, gpu.path) {
-							continue
-						}
-
-						err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, false)
-						if err != nil {
-							logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
-							return err
-						}
-					}
-				}
-
-				if !found {
-					msg := "Failed to detect requested GPU device"
-					logger.Error(msg)
-					return fmt.Errorf(msg)
-				}
 			}
 		}
 
@@ -6931,15 +6734,6 @@ func (c *containerLXC) removeMount(mount string) error {
 	return nil
 }
 
-// Check if the unix device already exists.
-func (c *containerLXC) deviceExistsInDevicesFolder(prefix string, path string) bool {
-	relativeDestPath := strings.TrimPrefix(path, "/")
-	devName := fmt.Sprintf("%s.%s", strings.Replace(prefix, "/", "-", -1), strings.Replace(relativeDestPath, "/", "-", -1))
-	devPath := filepath.Join(c.DevicesPath(), devName)
-
-	return shared.PathExists(devPath)
-}
-
 func (c *containerLXC) insertUnixDevice(prefix string, m config.Device, defaultMode bool) error {
 	// Check that the container is running
 	if !c.IsRunning() {
@@ -6952,12 +6746,12 @@ func (c *containerLXC) insertUnixDevice(prefix string, m config.Device, defaultM
 	}
 
 	// Create the device on the host
-	paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, m, defaultMode)
+	d, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, m, defaultMode)
 	if err != nil {
 		return fmt.Errorf("Failed to setup device: %s", err)
 	}
-	devPath := paths[0]
-	tgtPath := paths[1]
+	devPath := d.HostPath
+	tgtPath := d.RelativePath
 
 	// Bind-mount it into the container
 	err = c.insertMount(devPath, tgtPath, "none", unix.MS_BIND, false)
@@ -7050,12 +6844,12 @@ func (c *containerLXC) InsertSeccompUnixDevice(prefix string, m config.Device, p
 		return err
 	}
 
-	paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, m, true)
+	d, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, m, true)
 	if err != nil {
 		return fmt.Errorf("Failed to setup device: %s", err)
 	}
-	devPath := paths[0]
-	tgtPath := paths[1]
+	devPath := d.HostPath
+	tgtPath := d.RelativePath
 
 	// Bind-mount it into the container
 	defer os.Remove(devPath)


More information about the lxc-devel mailing list