[lxc-devel] [lxd/master] Device USB
tomponline on Github
lxc-bot at linuxcontainers.org
Wed Aug 14 16:50:03 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 435 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190814/bcc9abd6/attachment-0001.bin>
-------------- next part --------------
From 0cdae76953b8ad43216e5b6febd1a041645e086f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 17:55:55 +0100
Subject: [PATCH 01/26] devices: Removes gpu related code moved to device
package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/devices.go | 484 +------------------------------------------------
1 file changed, 3 insertions(+), 481 deletions(-)
diff --git a/lxd/devices.go b/lxd/devices.go
index b0d987e68e..2e962d4389 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -4,22 +4,17 @@ import (
"bufio"
"bytes"
"crypto/rand"
- "encoding/csv"
"fmt"
- "io"
"io/ioutil"
"math/big"
"os"
- "os/exec"
"path"
"path/filepath"
- "regexp"
"sort"
"strconv"
"strings"
"unsafe"
- "github.com/jaypipes/pcidb"
"golang.org/x/sys/unix"
"github.com/lxc/lxd/lxd/device"
@@ -66,479 +61,6 @@ type usbDevice struct {
ueventLen int
}
-// /dev/nvidia[0-9]+
-type nvidiaGpuCard struct {
- path string
- major int
- minor int
- id string
-
- nvrmVersion string
- cudaVersion string
- model string
- brand string
- uuid string
- architecture string
-}
-
-// {/dev/nvidiactl, /dev/nvidia-uvm, ...}
-type nvidiaGpuDevice struct {
- isCard bool
- path string
- major int
- minor int
-}
-
-// Nvidia container info
-type nvidiaContainerInfo struct {
- Cards map[string]*nvidiaContainerCardInfo
- NVRMVersion string
- CUDAVersion string
-}
-
-type nvidiaContainerCardInfo struct {
- DeviceIndex string
- DeviceMinor string
- Model string
- Brand string
- UUID string
- PCIAddress string
- Architecture string
-}
-
-// /dev/dri/card0. If we detect that vendor == nvidia, then nvidia will contain
-// the corresponding nvidia car, e.g. {/dev/dri/card1 to /dev/nvidia1}.
-type gpuDevice struct {
- // DRM node information
- id string
- path string
- major int
- minor int
-
- // Device information
- vendorID string
- vendorName string
- productID string
- productName string
- numaNode uint64
-
- // If related devices have the same PCI address as the GPU we should
- // mount them all. Meaning if we detect /dev/dri/card0,
- // /dev/dri/controlD64, and /dev/dri/renderD128 with the same PCI
- // address, then they should all be made available in the container.
- pci string
- driver string
- driverVersion string
-
- // NVIDIA specific handling
- isNvidia bool
- nvidia nvidiaGpuCard
-}
-
-func (g *gpuDevice) isNvidiaGpu() bool {
- return strings.EqualFold(g.vendorID, "10de")
-}
-
-type cardIds struct {
- id string
- pci string
-}
-
-// Fallback for old drivers which don't provide "Device Minor:"
-func findNvidiaMinorOld() (string, error) {
- var minor string
-
- // For now, just handle most common case (single nvidia card)
- ents, err := ioutil.ReadDir("/dev")
- if err != nil {
- return "", err
- }
-
- rp := regexp.MustCompile("^nvidia([0-9]+)$")
- for _, ent := range ents {
- matches := rp.FindStringSubmatch(ent.Name())
- if matches == nil {
- continue
- }
-
- if minor != "" {
- return "", fmt.Errorf("No device minor index detected, and more than one NVIDIA card present")
- }
- minor = matches[1]
- }
-
- if minor == "" {
- return "", fmt.Errorf("No device minor index detected, and no NVIDIA card present")
- }
-
- return minor, nil
-}
-
-// Return string for minor number of nvidia device corresponding to the given pci id
-func findNvidiaMinor(pci string) (string, error) {
- nvidiaPath := fmt.Sprintf("/proc/driver/nvidia/gpus/%s/information", pci)
- buf, err := ioutil.ReadFile(nvidiaPath)
- if err != nil {
- return "", err
- }
-
- strBuf := strings.TrimSpace(string(buf))
- idx := strings.Index(strBuf, "Device Minor:")
- if idx != -1 {
- idx += len("Device Minor:")
- strBuf = strBuf[idx:]
- strBuf = strings.TrimSpace(strBuf)
- parts := strings.SplitN(strBuf, "\n", 2)
- _, err = strconv.Atoi(parts[0])
- if err == nil {
- return parts[0], nil
- }
- }
-
- minor, err := findNvidiaMinorOld()
- if err == nil {
- return minor, nil
- }
-
- return "", err
-}
-
-func deviceWantsAllGPUs(m map[string]string) bool {
- return m["vendorid"] == "" && m["productid"] == "" && m["id"] == "" && m["pci"] == ""
-}
-
-func deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
- const DRM_PATH = "/sys/class/drm/"
- var gpus []gpuDevice
- var nvidiaDevices []nvidiaGpuDevice
- var cards []cardIds
-
- // Load NVIDIA information (if available)
- var nvidiaContainer *nvidiaContainerInfo
-
- _, err := exec.LookPath("nvidia-container-cli")
- if err == nil {
- out, err := shared.RunCommand("nvidia-container-cli", "info", "--csv")
- if err == nil {
- r := csv.NewReader(strings.NewReader(out))
- r.FieldsPerRecord = -1
-
- nvidiaContainer = &nvidiaContainerInfo{}
- nvidiaContainer.Cards = map[string]*nvidiaContainerCardInfo{}
- line := 0
- for {
- record, err := r.Read()
- if err == io.EOF {
- break
- }
- line += 1
-
- if err != nil {
- continue
- }
-
- if line == 2 && len(record) >= 2 {
- nvidiaContainer.NVRMVersion = record[0]
- nvidiaContainer.CUDAVersion = record[1]
- } else if line >= 4 {
- nvidiaContainer.Cards[record[5]] = &nvidiaContainerCardInfo{
- DeviceIndex: record[0],
- DeviceMinor: record[1],
- Model: record[2],
- Brand: record[3],
- UUID: record[4],
- PCIAddress: record[5],
- Architecture: record[6],
- }
- }
- }
- }
- }
-
- // Load PCI database
- pciDB, err := pcidb.New()
- if err != nil {
- pciDB = nil
- }
-
- // Get the list of DRM devices
- ents, err := ioutil.ReadDir(DRM_PATH)
- if err != nil {
- // No GPUs
- if os.IsNotExist(err) {
- return nil, nil, nil
- }
-
- return nil, nil, err
- }
-
- // Get the list of cards
- devices := []string{}
- for _, ent := range ents {
- dev, err := filepath.EvalSymlinks(fmt.Sprintf("%s/%s/device", DRM_PATH, ent.Name()))
- if err != nil {
- continue
- }
-
- if !shared.StringInSlice(dev, devices) {
- devices = append(devices, dev)
- }
- }
-
- isNvidia := false
- for _, device := range devices {
- // The pci address == the name of the directory. So let's use
- // this cheap way of retrieving it.
- pciAddr := filepath.Base(device)
-
- // Make sure that we are dealing with a GPU by looking whether
- // the "drm" subfolder exists.
- drm := filepath.Join(device, "drm")
- drmEnts, err := ioutil.ReadDir(drm)
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
- }
-
- // Retrieve vendor ID.
- vendorIdPath := filepath.Join(device, "vendor")
- vendorId, err := ioutil.ReadFile(vendorIdPath)
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
- }
-
- // Retrieve device ID.
- productIdPath := filepath.Join(device, "device")
- productId, err := ioutil.ReadFile(productIdPath)
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
- }
-
- // Retrieve node ID
- numaPath := fmt.Sprintf(filepath.Join(device, "numa_node"))
- numaNode := uint64(0)
- if shared.PathExists(numaPath) {
- numaID, err := shared.ParseNumberFromFile(numaPath)
- if err != nil {
- continue
- }
-
- if numaID > 0 {
- numaNode = uint64(numaID)
- }
- }
-
- // Retrieve driver
- driver := ""
- driverVersion := ""
- driverPath := filepath.Join(device, "driver")
- if shared.PathExists(driverPath) {
- target, err := os.Readlink(driverPath)
- if err != nil {
- continue
- }
-
- driver = filepath.Base(target)
-
- out, err := ioutil.ReadFile(filepath.Join(driverPath, "module", "version"))
- if err == nil {
- driverVersion = strings.TrimSpace(string(out))
- } else {
- uname, err := shared.Uname()
- if err != nil {
- continue
- }
- driverVersion = uname.Release
- }
- }
-
- // Store all associated subdevices, e.g. controlD64, renderD128.
- // The name of the directory == the last part of the
- // /dev/dri/controlD64 path. So drmEnt.Name() will give us
- // controlD64.
- for _, drmEnt := range drmEnts {
- vendorTmp := strings.TrimSpace(string(vendorId))
- productTmp := strings.TrimSpace(string(productId))
- vendorTmp = strings.TrimPrefix(vendorTmp, "0x")
- productTmp = strings.TrimPrefix(productTmp, "0x")
- tmpGpu := gpuDevice{
- pci: pciAddr,
- vendorID: vendorTmp,
- productID: productTmp,
- numaNode: numaNode,
- driver: driver,
- driverVersion: driverVersion,
- path: filepath.Join("/dev/dri", drmEnt.Name()),
- }
-
- // Fill vendor and product names
- if pciDB != nil {
- vendor, ok := pciDB.Vendors[tmpGpu.vendorID]
- if ok {
- tmpGpu.vendorName = vendor.Name
-
- for _, product := range vendor.Products {
- if product.ID == tmpGpu.productID {
- tmpGpu.productName = product.Name
- break
- }
- }
- }
- }
-
- majMinPath := filepath.Join(drm, drmEnt.Name(), "dev")
- majMinByte, err := ioutil.ReadFile(majMinPath)
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
- }
-
- majMin := strings.TrimSpace(string(majMinByte))
- majMinSlice := strings.Split(string(majMin), ":")
- if len(majMinSlice) != 2 {
- continue
- }
-
- majorInt, err := strconv.Atoi(majMinSlice[0])
- if err != nil {
- continue
- }
-
- minorInt, err := strconv.Atoi(majMinSlice[1])
- if err != nil {
- continue
- }
-
- tmpGpu.major = majorInt
- tmpGpu.minor = minorInt
-
- isCard, err := regexp.MatchString("^card[0-9]+", drmEnt.Name())
- if err != nil {
- continue
- }
-
- // Find matching /dev/nvidia* entry for /dev/dri/card*
- if tmpGpu.isNvidiaGpu() && isCard {
- if !isNvidia {
- isNvidia = true
- }
- tmpGpu.isNvidia = true
-
- if !all {
- minor, err := findNvidiaMinor(tmpGpu.pci)
- if err == nil {
- nvidiaPath := "/dev/nvidia" + minor
- stat := unix.Stat_t{}
- err = unix.Stat(nvidiaPath, &stat)
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
-
- return nil, nil, err
- }
-
- tmpGpu.nvidia.path = nvidiaPath
- tmpGpu.nvidia.major = shared.Major(stat.Rdev)
- tmpGpu.nvidia.minor = shared.Minor(stat.Rdev)
- tmpGpu.nvidia.id = strconv.Itoa(tmpGpu.nvidia.minor)
-
- if nvidiaContainer != nil {
- tmpGpu.nvidia.nvrmVersion = nvidiaContainer.NVRMVersion
- tmpGpu.nvidia.cudaVersion = nvidiaContainer.CUDAVersion
- nvidiaInfo, ok := nvidiaContainer.Cards[tmpGpu.pci]
- if !ok {
- nvidiaInfo, ok = nvidiaContainer.Cards[fmt.Sprintf("0000%v", tmpGpu.pci)]
- }
- if ok {
- tmpGpu.nvidia.brand = nvidiaInfo.Brand
- tmpGpu.nvidia.model = nvidiaInfo.Model
- tmpGpu.nvidia.uuid = nvidiaInfo.UUID
- tmpGpu.nvidia.architecture = nvidiaInfo.Architecture
- }
- }
- }
- }
- }
-
- if isCard {
- // If it is a card it's minor number will be its id.
- tmpGpu.id = strconv.Itoa(minorInt)
- tmp := cardIds{
- id: tmpGpu.id,
- pci: tmpGpu.pci,
- }
-
- cards = append(cards, tmp)
- }
-
- gpus = append(gpus, tmpGpu)
- }
- }
-
- // We detected a Nvidia card, so let's collect all other nvidia devices
- // that are not /dev/nvidia[0-9]+.
- if isNvidia {
- nvidiaEnts, err := ioutil.ReadDir("/dev")
- if err != nil {
- if os.IsNotExist(err) {
- return nil, nil, err
- }
- }
-
- validNvidia, err := regexp.Compile(`^nvidia[^0-9]+`)
- if err != nil {
- return nil, nil, err
- }
-
- for _, nvidiaEnt := range nvidiaEnts {
- if all {
- if !strings.HasPrefix(nvidiaEnt.Name(), "nvidia") {
- continue
- }
- } else {
- if !validNvidia.MatchString(nvidiaEnt.Name()) {
- continue
- }
- }
-
- nvidiaPath := filepath.Join("/dev", nvidiaEnt.Name())
- stat := unix.Stat_t{}
- err = unix.Stat(nvidiaPath, &stat)
- if err != nil {
- continue
- }
-
- tmpNividiaGpu := nvidiaGpuDevice{
- isCard: !validNvidia.MatchString(nvidiaEnt.Name()),
- path: nvidiaPath,
- major: shared.Major(stat.Rdev),
- minor: shared.Minor(stat.Rdev),
- }
-
- nvidiaDevices = append(nvidiaDevices, tmpNividiaGpu)
- }
- }
-
- // Since we'll give users to ability to specify and id we need to group
- // devices on the same PCI that belong to the same card by id.
- for _, card := range cards {
- for i := 0; i < len(gpus); i++ {
- if gpus[i].pci == card.pci {
- gpus[i].id = card.id
- }
- }
- }
-
- return gpus, nvidiaDevices, nil
-}
-
func createUSBDevice(action string, vendor string, product string, major string, minor string, busnum string, devnum string, devname string, ueventParts []string, ueventLen int) (usbDevice, error) {
majorInt, err := strconv.Atoi(major)
if err != nil {
@@ -1279,7 +801,7 @@ func deviceGetParentBlocks(path string) ([]string, error) {
}
if path != "" {
- _, major, minor, err := device.UnixGetDeviceAttributes(path)
+ _, major, minor, err := device.UnixDeviceAttributes(path)
if err != nil {
continue
}
@@ -1304,7 +826,7 @@ func deviceGetParentBlocks(path string) ([]string, error) {
continue
}
- _, major, minor, err := device.UnixGetDeviceAttributes(fields[len(fields)-1])
+ _, major, minor, err := device.UnixDeviceAttributes(fields[len(fields)-1])
if err != nil {
return nil, err
}
@@ -1313,7 +835,7 @@ func deviceGetParentBlocks(path string) ([]string, error) {
}
} else if shared.PathExists(dev[1]) {
// Anything else with a valid path
- _, major, minor, err := device.UnixGetDeviceAttributes(dev[1])
+ _, major, minor, err := device.UnixDeviceAttributes(dev[1])
if err != nil {
return nil, err
}
From 743b4b62072a05eed9629d981885750fb8442390 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 17:55:32 +0100
Subject: [PATCH 02/26] device/gpu: Adds gpu implementation
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/gpu.go | 644 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 644 insertions(+)
create mode 100644 lxd/device/gpu.go
diff --git a/lxd/device/gpu.go b/lxd/device/gpu.go
new file mode 100644
index 0000000000..3ede69e6c9
--- /dev/null
+++ b/lxd/device/gpu.go
@@ -0,0 +1,644 @@
+package device
+
+import (
+ "encoding/csv"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "github.com/jaypipes/pcidb"
+ "golang.org/x/sys/unix"
+
+ "github.com/lxc/lxd/lxd/device/config"
+ "github.com/lxc/lxd/lxd/instance"
+ "github.com/lxc/lxd/shared"
+)
+
+type gpu struct {
+ deviceCommon
+}
+
+// /dev/dri/card0. If we detect that vendor == nvidia, then nvidia will contain
+// the corresponding nvidia car, e.g. {/dev/dri/card1 to /dev/nvidia1}.
+type gpuDevice struct {
+ // DRM node information
+ id string
+ path string
+ major int
+ minor int
+
+ // Device information
+ vendorID string
+ vendorName string
+ productID string
+ productName string
+ numaNode uint64
+
+ // If related devices have the same PCI address as the GPU we should
+ // mount them all. Meaning if we detect /dev/dri/card0,
+ // /dev/dri/controlD64, and /dev/dri/renderD128 with the same PCI
+ // address, then they should all be made available in the container.
+ pci string
+ driver string
+ driverVersion string
+
+ // NVIDIA specific handling
+ isNvidia bool
+ nvidia nvidiaGpuCard
+}
+
+func (g *gpuDevice) isNvidiaGpu() bool {
+ return strings.EqualFold(g.vendorID, "10de")
+}
+
+// /dev/nvidia[0-9]+
+type nvidiaGpuCard struct {
+ path string
+ major int
+ minor int
+ id string
+
+ nvrmVersion string
+ cudaVersion string
+ model string
+ brand string
+ uuid string
+ architecture string
+}
+
+// {/dev/nvidiactl, /dev/nvidia-uvm, ...}
+type nvidiaGpuDevice struct {
+ isCard bool
+ path string
+ major int
+ minor int
+}
+
+// Nvidia container info
+type nvidiaContainerInfo struct {
+ Cards map[string]*nvidiaContainerCardInfo
+ NVRMVersion string
+ CUDAVersion string
+}
+
+type nvidiaContainerCardInfo struct {
+ DeviceIndex string
+ DeviceMinor string
+ Model string
+ Brand string
+ UUID string
+ PCIAddress string
+ Architecture string
+}
+
+type cardIds struct {
+ id string
+ pci string
+}
+
+// validateConfig checks the supplied config for correctness.
+func (d *gpu) validateConfig() error {
+ if d.instance.Type() != instance.TypeContainer {
+ return ErrUnsupportedDevType
+ }
+
+ rules := map[string]func(string) error{
+ "vendorid": shared.IsAny,
+ "productid": shared.IsAny,
+ "id": shared.IsAny,
+ "pci": shared.IsAny,
+ "uid": shared.IsUnixUserID,
+ "gid": shared.IsUnixUserID,
+ "mode": shared.IsOctalFileMode,
+ }
+
+ err := config.ValidateDevice(rules, d.config)
+ if err != nil {
+ return err
+ }
+
+ if d.config["pci"] != "" && (d.config["id"] != "" || d.config["productid"] != "" || d.config["vendorid"] != "") {
+ return fmt.Errorf("Cannot use id, productid or vendorid when pci is set")
+ }
+
+ if d.config["id"] != "" && (d.config["pci"] != "" || d.config["productid"] != "" || d.config["vendorid"] != "") {
+ return fmt.Errorf("Cannot use pci, productid or vendorid when id is set")
+ }
+
+ return nil
+}
+
+// validateEnvironment checks the runtime environment for correctness.
+func (d *gpu) validateEnvironment() error {
+ if d.config["pci"] != "" && !shared.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s", d.config["pci"])) {
+ return fmt.Errorf("Invalid PCI address (no device found): %s", d.config["pci"])
+ }
+
+ return nil
+}
+
+// Start is run when the device is added to the container.
+func (d *gpu) Start() (*RunConfig, error) {
+ err := d.validateEnvironment()
+ if err != nil {
+ return nil, err
+ }
+
+ runConf := RunConfig{}
+
+ allGpus := d.deviceWantsAllGPUs(d.config)
+ gpus, nvidiaDevices, err := d.deviceLoadGpu(allGpus)
+ if err != nil {
+ return nil, err
+ }
+
+ sawNvidia := false
+ found := false
+ for _, gpu := range gpus {
+ if (d.config["vendorid"] != "" && gpu.vendorID != d.config["vendorid"]) ||
+ (d.config["pci"] != "" && gpu.pci != d.config["pci"]) ||
+ (d.config["productid"] != "" && gpu.productID != d.config["productid"]) ||
+ (d.config["id"] != "" && gpu.id != d.config["id"]) {
+ continue
+ }
+
+ found = true
+ err := unixDeviceSetupCharNum(d.state, d.instance.DevicesPath(), "unix", d.name, d.config, gpu.major, gpu.minor, gpu.path, false, &runConf)
+ if err != nil {
+ return nil, err
+ }
+
+ if !gpu.isNvidia {
+ continue
+ }
+
+ if gpu.nvidia.path != "" {
+ err = unixDeviceSetupCharNum(d.state, d.instance.DevicesPath(), "unix", d.name, d.config, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path, false, &runConf)
+ if err != nil {
+ return nil, err
+ }
+ } else if !allGpus {
+ return nil, fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
+ }
+
+ sawNvidia = true
+ }
+
+ if sawNvidia {
+ for _, gpu := range nvidiaDevices {
+ instanceConfig := d.instance.ExpandedConfig()
+
+ // No need to mount additional nvidia non-card devices as the nvidia.runtime
+ // setting will do this for us.
+ if shared.IsTrue(instanceConfig["nvidia.runtime"]) {
+ if !gpu.isCard {
+ continue
+ }
+ }
+
+ prefix := unixDeviceJoinPath("unix", d.name)
+ if UnixDeviceExists(d.instance.DevicesPath(), prefix, gpu.path) {
+ continue
+ }
+
+ err = unixDeviceSetupCharNum(d.state, d.instance.DevicesPath(), "unix", d.name, d.config, gpu.major, gpu.minor, gpu.path, false, &runConf)
+ if err != nil {
+ return nil, err
+ }
+ }
+ }
+
+ if !found {
+ return nil, fmt.Errorf("Failed to detect requested GPU device")
+ }
+
+ return &runConf, nil
+}
+
+// Stop is run when the device is removed from the instance.
+func (d *gpu) Stop() (*RunConfig, error) {
+ runConf := RunConfig{
+ PostHooks: []func() error{d.postStop},
+ }
+
+ err := unixDeviceRemove(d.instance.DevicesPath(), "unix", d.name, &runConf)
+ if err != nil {
+ return nil, err
+ }
+
+ return &runConf, nil
+}
+
+// postStop is run after the device is removed from the instance.
+func (d *gpu) postStop() error {
+ // Remove host files for this device.
+ err := unixDeviceDeleteFiles(d.state, d.instance.DevicesPath(), "unix", d.name)
+ if err != nil {
+ return fmt.Errorf("Failed to delete files for device '%s': %v", d.name, err)
+ }
+
+ return nil
+}
+
+// deviceWantsAllGPUs whether the LXD device wants to passthrough all GPUs on the host.
+func (d *gpu) deviceWantsAllGPUs(m map[string]string) bool {
+ return m["vendorid"] == "" && m["productid"] == "" && m["id"] == "" && m["pci"] == ""
+}
+
+// deviceLoadGpu probes the system for information about the available GPUs.
+func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
+ const drmPath = "/sys/class/drm/"
+ var gpus []gpuDevice
+ var nvidiaDevices []nvidiaGpuDevice
+ var cards []cardIds
+
+ // Load NVIDIA information (if available)
+ var nvidiaContainer *nvidiaContainerInfo
+
+ _, err := exec.LookPath("nvidia-container-cli")
+ if err == nil {
+ out, err := shared.RunCommand("nvidia-container-cli", "info", "--csv")
+ if err == nil {
+ r := csv.NewReader(strings.NewReader(out))
+ r.FieldsPerRecord = -1
+
+ nvidiaContainer = &nvidiaContainerInfo{}
+ nvidiaContainer.Cards = map[string]*nvidiaContainerCardInfo{}
+ line := 0
+ for {
+ record, err := r.Read()
+ if err == io.EOF {
+ break
+ }
+ line++
+
+ if err != nil {
+ continue
+ }
+
+ if line == 2 && len(record) >= 2 {
+ nvidiaContainer.NVRMVersion = record[0]
+ nvidiaContainer.CUDAVersion = record[1]
+ } else if line >= 4 {
+ nvidiaContainer.Cards[record[5]] = &nvidiaContainerCardInfo{
+ DeviceIndex: record[0],
+ DeviceMinor: record[1],
+ Model: record[2],
+ Brand: record[3],
+ UUID: record[4],
+ PCIAddress: record[5],
+ Architecture: record[6],
+ }
+ }
+ }
+ }
+ }
+
+ // Load PCI database
+ pciDB, err := pcidb.New()
+ if err != nil {
+ pciDB = nil
+ }
+
+ // Get the list of DRM devices
+ ents, err := ioutil.ReadDir(drmPath)
+ if err != nil {
+ // No GPUs
+ if os.IsNotExist(err) {
+ return nil, nil, nil
+ }
+
+ return nil, nil, err
+ }
+
+ // Get the list of cards
+ devices := []string{}
+ for _, ent := range ents {
+ dev, err := filepath.EvalSymlinks(fmt.Sprintf("%s/%s/device", drmPath, ent.Name()))
+ if err != nil {
+ continue
+ }
+
+ if !shared.StringInSlice(dev, devices) {
+ devices = append(devices, dev)
+ }
+ }
+
+ isNvidia := false
+ for _, device := range devices {
+ // The pci address == the name of the directory. So let's use
+ // this cheap way of retrieving it.
+ pciAddr := filepath.Base(device)
+
+ // Make sure that we are dealing with a GPU by looking whether
+ // the "drm" subfolder exists.
+ drm := filepath.Join(device, "drm")
+ drmEnts, err := ioutil.ReadDir(drm)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+ }
+
+ // Retrieve vendor ID.
+ vendorIDPath := filepath.Join(device, "vendor")
+ vendorID, err := ioutil.ReadFile(vendorIDPath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+ }
+
+ // Retrieve device ID.
+ productIDPath := filepath.Join(device, "device")
+ productID, err := ioutil.ReadFile(productIDPath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+ }
+
+ // Retrieve node ID
+ numaPath := fmt.Sprintf(filepath.Join(device, "numa_node"))
+ numaNode := uint64(0)
+ if shared.PathExists(numaPath) {
+ numaID, err := shared.ParseNumberFromFile(numaPath)
+ if err != nil {
+ continue
+ }
+
+ if numaID > 0 {
+ numaNode = uint64(numaID)
+ }
+ }
+
+ // Retrieve driver
+ driver := ""
+ driverVersion := ""
+ driverPath := filepath.Join(device, "driver")
+ if shared.PathExists(driverPath) {
+ target, err := os.Readlink(driverPath)
+ if err != nil {
+ continue
+ }
+
+ driver = filepath.Base(target)
+
+ out, err := ioutil.ReadFile(filepath.Join(driverPath, "module", "version"))
+ if err == nil {
+ driverVersion = strings.TrimSpace(string(out))
+ } else {
+ uname, err := shared.Uname()
+ if err != nil {
+ continue
+ }
+ driverVersion = uname.Release
+ }
+ }
+
+ // Store all associated subdevices, e.g. controlD64, renderD128.
+ // The name of the directory == the last part of the
+ // /dev/dri/controlD64 path. So drmEnt.Name() will give us
+ // controlD64.
+ for _, drmEnt := range drmEnts {
+ vendorTmp := strings.TrimSpace(string(vendorID))
+ productTmp := strings.TrimSpace(string(productID))
+ vendorTmp = strings.TrimPrefix(vendorTmp, "0x")
+ productTmp = strings.TrimPrefix(productTmp, "0x")
+ tmpGpu := gpuDevice{
+ pci: pciAddr,
+ vendorID: vendorTmp,
+ productID: productTmp,
+ numaNode: numaNode,
+ driver: driver,
+ driverVersion: driverVersion,
+ path: filepath.Join("/dev/dri", drmEnt.Name()),
+ }
+
+ // Fill vendor and product names
+ if pciDB != nil {
+ vendor, ok := pciDB.Vendors[tmpGpu.vendorID]
+ if ok {
+ tmpGpu.vendorName = vendor.Name
+
+ for _, product := range vendor.Products {
+ if product.ID == tmpGpu.productID {
+ tmpGpu.productName = product.Name
+ break
+ }
+ }
+ }
+ }
+
+ majMinPath := filepath.Join(drm, drmEnt.Name(), "dev")
+ majMinByte, err := ioutil.ReadFile(majMinPath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+ }
+
+ majMin := strings.TrimSpace(string(majMinByte))
+ majMinSlice := strings.Split(string(majMin), ":")
+ if len(majMinSlice) != 2 {
+ continue
+ }
+
+ majorInt, err := strconv.Atoi(majMinSlice[0])
+ if err != nil {
+ continue
+ }
+
+ minorInt, err := strconv.Atoi(majMinSlice[1])
+ if err != nil {
+ continue
+ }
+
+ tmpGpu.major = majorInt
+ tmpGpu.minor = minorInt
+
+ isCard, err := regexp.MatchString("^card[0-9]+", drmEnt.Name())
+ if err != nil {
+ continue
+ }
+
+ // Find matching /dev/nvidia* entry for /dev/dri/card*
+ if tmpGpu.isNvidiaGpu() && isCard {
+ if !isNvidia {
+ isNvidia = true
+ }
+ tmpGpu.isNvidia = true
+
+ if !all {
+ minor, err := d.findNvidiaMinor(tmpGpu.pci)
+ if err == nil {
+ nvidiaPath := "/dev/nvidia" + minor
+ stat := unix.Stat_t{}
+ err = unix.Stat(nvidiaPath, &stat)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+
+ return nil, nil, err
+ }
+
+ tmpGpu.nvidia.path = nvidiaPath
+ tmpGpu.nvidia.major = shared.Major(stat.Rdev)
+ tmpGpu.nvidia.minor = shared.Minor(stat.Rdev)
+ tmpGpu.nvidia.id = strconv.Itoa(tmpGpu.nvidia.minor)
+
+ if nvidiaContainer != nil {
+ tmpGpu.nvidia.nvrmVersion = nvidiaContainer.NVRMVersion
+ tmpGpu.nvidia.cudaVersion = nvidiaContainer.CUDAVersion
+ nvidiaInfo, ok := nvidiaContainer.Cards[tmpGpu.pci]
+ if !ok {
+ nvidiaInfo, ok = nvidiaContainer.Cards[fmt.Sprintf("0000%v", tmpGpu.pci)]
+ }
+ if ok {
+ tmpGpu.nvidia.brand = nvidiaInfo.Brand
+ tmpGpu.nvidia.model = nvidiaInfo.Model
+ tmpGpu.nvidia.uuid = nvidiaInfo.UUID
+ tmpGpu.nvidia.architecture = nvidiaInfo.Architecture
+ }
+ }
+ }
+ }
+ }
+
+ if isCard {
+ // If it is a card it's minor number will be its id.
+ tmpGpu.id = strconv.Itoa(minorInt)
+ tmp := cardIds{
+ id: tmpGpu.id,
+ pci: tmpGpu.pci,
+ }
+
+ cards = append(cards, tmp)
+ }
+
+ gpus = append(gpus, tmpGpu)
+ }
+ }
+
+ // We detected a Nvidia card, so let's collect all other nvidia devices
+ // that are not /dev/nvidia[0-9]+.
+ if isNvidia {
+ nvidiaEnts, err := ioutil.ReadDir("/dev")
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil, err
+ }
+ }
+
+ validNvidia, err := regexp.Compile(`^nvidia[^0-9]+`)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ for _, nvidiaEnt := range nvidiaEnts {
+ if all {
+ if !strings.HasPrefix(nvidiaEnt.Name(), "nvidia") {
+ continue
+ }
+ } else {
+ if !validNvidia.MatchString(nvidiaEnt.Name()) {
+ continue
+ }
+ }
+
+ nvidiaPath := filepath.Join("/dev", nvidiaEnt.Name())
+ stat := unix.Stat_t{}
+ err = unix.Stat(nvidiaPath, &stat)
+ if err != nil {
+ continue
+ }
+
+ tmpNividiaGpu := nvidiaGpuDevice{
+ isCard: !validNvidia.MatchString(nvidiaEnt.Name()),
+ path: nvidiaPath,
+ major: shared.Major(stat.Rdev),
+ minor: shared.Minor(stat.Rdev),
+ }
+
+ nvidiaDevices = append(nvidiaDevices, tmpNividiaGpu)
+ }
+ }
+
+ // Since we'll give users to ability to specify and id we need to group
+ // devices on the same PCI that belong to the same card by id.
+ for _, card := range cards {
+ for i := 0; i < len(gpus); i++ {
+ if gpus[i].pci == card.pci {
+ gpus[i].id = card.id
+ }
+ }
+ }
+
+ return gpus, nvidiaDevices, nil
+}
+
+// findNvidiaMinorOld fallback for old drivers which don't provide "Device Minor:".
+func (d *gpu) findNvidiaMinorOld() (string, error) {
+ var minor string
+
+ // For now, just handle most common case (single nvidia card)
+ ents, err := ioutil.ReadDir("/dev")
+ if err != nil {
+ return "", err
+ }
+
+ rp := regexp.MustCompile("^nvidia([0-9]+)$")
+ for _, ent := range ents {
+ matches := rp.FindStringSubmatch(ent.Name())
+ if matches == nil {
+ continue
+ }
+
+ if minor != "" {
+ return "", fmt.Errorf("No device minor index detected, and more than one NVIDIA card present")
+ }
+ minor = matches[1]
+ }
+
+ if minor == "" {
+ return "", fmt.Errorf("No device minor index detected, and no NVIDIA card present")
+ }
+
+ return minor, nil
+}
+
+// findNvidiaMinor returns minor number of nvidia device corresponding to the given pci id.
+func (d *gpu) findNvidiaMinor(pci string) (string, error) {
+ nvidiaPath := fmt.Sprintf("/proc/driver/nvidia/gpus/%s/information", pci)
+ buf, err := ioutil.ReadFile(nvidiaPath)
+ if err != nil {
+ return "", err
+ }
+
+ strBuf := strings.TrimSpace(string(buf))
+ idx := strings.Index(strBuf, "Device Minor:")
+ if idx != -1 {
+ idx += len("Device Minor:")
+ strBuf = strBuf[idx:]
+ strBuf = strings.TrimSpace(strBuf)
+ parts := strings.SplitN(strBuf, "\n", 2)
+ _, err = strconv.Atoi(parts[0])
+ if err == nil {
+ return parts[0], nil
+ }
+ }
+
+ minor, err := d.findNvidiaMinorOld()
+ if err == nil {
+ return minor, nil
+ }
+
+ return "", err
+}
From accb85f6a8afc4b9f8098e937a2b948047d2ab5b Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:32:43 +0100
Subject: [PATCH 03/26] container: Removes gpu validation as moved to device
package
And UnixDeviceAttributes rename
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container.go | 33 +--------------------------------
1 file changed, 1 insertion(+), 32 deletions(-)
diff --git a/lxd/container.go b/lxd/container.go
index c3c874c294..5b795f3d55 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -194,25 +194,6 @@ func containerValidDeviceConfigKey(t, k string) bool {
default:
return false
}
- case "gpu":
- switch k {
- case "vendorid":
- return true
- case "productid":
- return true
- case "id":
- return true
- case "pci":
- return true
- case "mode":
- return true
- case "gid":
- return true
- case "uid":
- return true
- default:
- return false
- }
case "none":
return false
default:
@@ -410,7 +391,7 @@ func containerValidDevices(state *state.State, cluster *db.Cluster, devices conf
return fmt.Errorf("The device path doesn't exist on the host and major/minor wasn't specified")
}
- dType, _, _, err := device.UnixGetDeviceAttributes(srcPath)
+ dType, _, _, err := device.UnixDeviceAttributes(srcPath)
if err != nil {
return err
}
@@ -425,18 +406,6 @@ func containerValidDevices(state *state.State, cluster *db.Cluster, devices conf
}
} else if m["type"] == "usb" {
// Nothing needed for usb.
- } else if m["type"] == "gpu" {
- if m["pci"] != "" && !shared.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s", m["pci"])) {
- return fmt.Errorf("Invalid PCI address (no device found): %s", m["pci"])
- }
-
- if m["pci"] != "" && (m["id"] != "" || m["productid"] != "" || m["vendorid"] != "") {
- return fmt.Errorf("Cannot use id, productid or vendorid when pci is set")
- }
-
- if m["id"] != "" && (m["pci"] != "" || m["productid"] != "" || m["vendorid"] != "") {
- return fmt.Errorf("Cannot use pci, productid or vendorid when id is set")
- }
} else if m["type"] == "none" {
continue
} else {
From a034ea32f41173f4efbb7975608e5ba02570766a Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:35:37 +0100
Subject: [PATCH 04/26] device: Links gpu device
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/lxd/device/device.go b/lxd/device/device.go
index a7566dda42..5c6833e1e6 100644
--- a/lxd/device/device.go
+++ b/lxd/device/device.go
@@ -10,8 +10,9 @@ import (
// devTypes defines supported top-level device type creation functions.
var devTypes = map[string]func(config.Device) device{
"nic": nicLoadByType,
- "proxy": func(c config.Device) device { return &proxy{} },
"infiniband": infinibandLoadByType,
+ "proxy": func(c config.Device) device { return &proxy{} },
+ "gpu": func(c config.Device) device { return &gpu{} },
}
// VolatileSetter is a function that accepts one or more key/value strings to save into the LXD
From 7653e4ba6abddba005290adad77c43a8ecb5ad8f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:36:22 +0100
Subject: [PATCH 05/26] device/device/utils/infiniband: Updates use of unix
functions
The unix device create functions have been updated to return info about the device created, which simplifies some of the infiniband setup.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_infiniband.go | 251 +-------------------------
1 file changed, 2 insertions(+), 249 deletions(-)
diff --git a/lxd/device/device_utils_infiniband.go b/lxd/device/device_utils_infiniband.go
index 01d606854b..57f2f2e2c5 100644
--- a/lxd/device/device_utils_infiniband.go
+++ b/lxd/device/device_utils_infiniband.go
@@ -5,12 +5,9 @@ import (
"fmt"
"io/ioutil"
"os"
- "path/filepath"
"strconv"
"strings"
- "golang.org/x/sys/unix"
-
"github.com/lxc/lxd/lxd/device/config"
"github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/shared"
@@ -260,272 +257,28 @@ func infinibandLoadDevices() (map[string]IBF, error) {
// infinibandAddDevices creates the UNIX devices for the provided IBF device and then configures the
// supplied runConfig with the Cgroup rules and mount instructions to pass the device into instance.
func infinibandAddDevices(s *state.State, devicesPath string, deviceName string, ifDev *IBF, runConf *RunConfig) error {
- err := infinibandAddDevicesPerPort(s, devicesPath, deviceName, ifDev, runConf)
- if err != nil {
- return err
- }
-
- err = infinibandAddDevicesPerFun(s, devicesPath, deviceName, ifDev, runConf)
- if err != nil {
- return err
- }
-
- return nil
-}
-
-func infinibandAddDevicesPerPort(s *state.State, devicesPath string, deviceName string, ifDev *IBF, runConf *RunConfig) error {
for _, unixCharDev := range ifDev.PerPortDevices {
destPath := fmt.Sprintf("/dev/infiniband/%s", unixCharDev)
- relDestPath := destPath[1:]
- devPrefix := fmt.Sprintf("%s.%s", IBDevPrefix, deviceName)
-
- // Unix device.
dummyDevice := config.Device{
"source": destPath,
}
- deviceExists := false
-
- // Only handle "infiniband.unix." devices.
- prefix := fmt.Sprintf("%s.", IBDevPrefix)
-
- // Load all devices.
- dents, err := ioutil.ReadDir(devicesPath)
- if err != nil {
- if !os.IsNotExist(err) {
- return err
- }
- }
-
- for _, ent := range dents {
- // Skip non "infiniband.unix." devices.
- devName := ent.Name()
- if !strings.HasPrefix(devName, prefix) {
- continue
- }
-
- // Extract the path inside the container.
- idx := strings.LastIndex(devName, ".")
- if idx == -1 {
- return fmt.Errorf("Invalid infiniband device name \"%s\"", devName)
- }
- rPath := devName[idx+1:]
- rPath = strings.Replace(rPath, "-", "/", -1)
- if rPath != relDestPath {
- continue
- }
-
- deviceExists = true
- break
- }
-
- paths, err := UnixCreateDevice(s, nil, devicesPath, devPrefix, dummyDevice, false)
- if err != nil {
- return err
- }
- devPath := paths[0]
-
- // If an existing device with the same path in the instance exists, then do not
- // request it be mounted again.
- if deviceExists {
- continue
- }
-
- // Instruct liblxc to perform the mount.
- runConf.Mounts = append(runConf.Mounts, MountEntryItem{
- DevPath: devPath,
- TargetPath: relDestPath,
- FSType: "none",
- Opts: []string{"bind", "create=file"},
- })
-
- // Add the new device cgroup rule.
- dType, dMajor, dMinor, err := UnixGetDeviceAttributes(devPath)
+ err := unixDeviceSetup(s, devicesPath, IBDevPrefix, deviceName, dummyDevice, false, runConf)
if err != nil {
return err
}
-
- // Instruct liblxc to setup the cgroup rule.
- runConf.CGroups = append(runConf.CGroups, RunConfigItem{
- Key: "devices.allow",
- Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor),
- })
}
- return nil
-}
-
-func infinibandAddDevicesPerFun(s *state.State, devicesPath string, deviceName string, ifDev *IBF, runConf *RunConfig) error {
for _, unixCharDev := range ifDev.PerFunDevices {
destPath := fmt.Sprintf("/dev/infiniband/%s", unixCharDev)
- uniqueDevPrefix := fmt.Sprintf("%s.%s", IBDevPrefix, deviceName)
- relativeDestPath := fmt.Sprintf("dev/infiniband/%s", unixCharDev)
- uniqueDevName := fmt.Sprintf("%s.%s", uniqueDevPrefix, strings.Replace(relativeDestPath, "/", "-", -1))
- hostDevPath := filepath.Join(devicesPath, uniqueDevName)
-
dummyDevice := config.Device{
"source": destPath,
}
- // Instruct liblxc to perform the mount.
- runConf.Mounts = append(runConf.Mounts, MountEntryItem{
- DevPath: hostDevPath,
- TargetPath: relativeDestPath,
- FSType: "none",
- Opts: []string{"bind", "create=file"},
- })
-
- paths, err := UnixCreateDevice(s, nil, devicesPath, uniqueDevPrefix, dummyDevice, false)
+ err := unixDeviceSetup(s, devicesPath, IBDevPrefix, deviceName, dummyDevice, false, runConf)
if err != nil {
return err
}
- devPath := paths[0]
-
- // Add the new device cgroup rule.
- dType, dMajor, dMinor, err := UnixGetDeviceAttributes(devPath)
- if err != nil {
- return err
- }
-
- // Instruct liblxc to setup the cgroup rule.
- runConf.CGroups = append(runConf.CGroups, RunConfigItem{
- Key: "devices.allow",
- Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor),
- })
- }
-
- return nil
-}
-
-// infinibandRemoveDevices identifies all UNIX devices related to the supplied deviceName and then
-// populates the supplied runConf with the instructions to remove cgroup rules and unmount devices.
-func infinibandRemoveDevices(devicesPath string, deviceName string, runConf *RunConfig) error {
- // Load all devices.
- dents, err := ioutil.ReadDir(devicesPath)
- if err != nil {
- if !os.IsNotExist(err) {
- return err
- }
- }
-
- prefix := fmt.Sprintf("%s.", IBDevPrefix)
- ourPrefix := fmt.Sprintf("%s.%s", IBDevPrefix, deviceName)
- ourIFBDevs := []string{}
- otherIFBDevs := []string{}
-
- for _, ent := range dents {
- // Skip non "infiniband.unix." devices.
- devName := ent.Name()
- if !strings.HasPrefix(devName, prefix) {
- continue
- }
-
- // This is our infiniband device.
- if strings.HasPrefix(devName, ourPrefix) {
- ourIFBDevs = append(ourIFBDevs, devName)
- continue
- }
-
- // This someone else's infiniband device.
- otherIFBDevs = append(otherIFBDevs, devName)
- }
-
- // With infiniband it is possible for multiple LXD configured devices to share the same
- // UNIX character device for memory access, as char devices might be per port not per device.
- // Because of this, before we setup instructions to umount the device from the instance we
- // need to check whether any other LXD devices also use the same char device inside the
- // instance. To do this, scan all devices for this instance and use the dev path encoded
- // in the host-side filename to check if any match.
- residualInfinibandDevs := []string{}
- for _, otherIFBDev := range otherIFBDevs {
- idx := strings.LastIndex(otherIFBDev, ".")
- if idx == -1 {
- return fmt.Errorf("Invalid infiniband device name \"%s\"", otherIFBDev)
- }
- // Remove the LXD device name prefix, so we're left with dev path inside the instance.
- relPeerPath := otherIFBDev[idx+1:]
- relPeerPath = strings.Replace(relPeerPath, "-", "/", -1)
- absPeerPath := fmt.Sprintf("/%s", relPeerPath)
- residualInfinibandDevs = append(residualInfinibandDevs, absPeerPath)
- }
-
- // Check that none of our infiniband devices are in use by another LXD device.
- for _, ourDev := range ourIFBDevs {
- idx := strings.LastIndex(ourDev, ".")
- if idx == -1 {
- return fmt.Errorf("Invalid infiniband device name \"%s\"", ourDev)
- }
- rPath := ourDev[idx+1:]
- rPath = strings.Replace(rPath, "-", "/", -1)
- absPath := fmt.Sprintf("/%s", rPath)
- dupe := false
-
- // Look for infiniband devices for other LXD devices that match the same path.
- for _, peerDevPath := range residualInfinibandDevs {
- if peerDevPath == absPath {
- dupe = true
- break
- }
- }
-
- // If a device has been found that points to the same device inside the instance
- // then we cannot request it be umounted inside the instance as it's still in use.
- if dupe {
- continue
- }
-
- // Append this device to the mount rules (these will be unmounted).
- runConf.Mounts = append(runConf.Mounts, MountEntryItem{
- TargetPath: rPath,
- })
-
- dummyDevice := config.Device{
- "source": absPath,
- }
-
- dType, dMajor, dMinor, err := instanceUnixGetDeviceAttributes(devicesPath, ourPrefix, dummyDevice)
- if err != nil {
- return err
- }
-
- // Append a deny cgroup fule for this device.
- runConf.CGroups = append(runConf.CGroups, RunConfigItem{
- Key: "devices.deny",
- Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor),
- })
- }
-
- return nil
-}
-
-func infinibandDeleteHostFiles(s *state.State, devicesPath string, deviceName string) error {
- // Load all devices.
- dents, err := ioutil.ReadDir(devicesPath)
- if err != nil {
- if !os.IsNotExist(err) {
- return err
- }
- }
-
- // Remove our host side device files.
- ourPrefix := fmt.Sprintf("%s.%s", IBDevPrefix, deviceName)
- for _, ent := range dents {
- devName := ent.Name()
- devPath := filepath.Join(devicesPath, devName)
-
- // Check this is our infiniband device.
- if strings.HasPrefix(devName, ourPrefix) {
- // Remove the host side mount.
- if s.OS.RunningInUserNS {
- unix.Unmount(devPath, unix.MNT_DETACH)
- }
-
- // Remove the host side device file.
- err = os.Remove(devPath)
- if err != nil {
- return err
- }
- }
}
return nil
From f569b1d8520d453323bf514c0c79051f11fa91cc Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:38:18 +0100
Subject: [PATCH 06/26] device/device/utils/unix: Device management function
rework
Updates UnixDeviceCreate() to return a UnixDevice struct with all the info about the created device.
This is to avoid having to have duplicate code elsewhere to stat the device to find out about it's properties.
Also adds unixDeviceSetup() and unixDeviceSetupCharNum() helper functions that will call UnixDeviceSetup() and then use the returned information to configure the RunConfig struct to instruct LXD to attach the device to the instance.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_unix.go | 368 ++++++++++++++++++++++++++++----
1 file changed, 325 insertions(+), 43 deletions(-)
diff --git a/lxd/device/device_utils_unix.go b/lxd/device/device_utils_unix.go
index 45c816494a..67e96a44fd 100644
--- a/lxd/device/device_utils_unix.go
+++ b/lxd/device/device_utils_unix.go
@@ -2,6 +2,7 @@ package device
import (
"fmt"
+ "io/ioutil"
"os"
"path/filepath"
"strconv"
@@ -16,10 +17,10 @@ import (
"github.com/lxc/lxd/shared/logger"
)
-// instanceUnixGetDeviceAttributes returns the UNIX device attributes for an instance device.
+// unixDeviceInstanceAttributes returns the UNIX device attributes for an instance device.
// Uses supplied device config for device properties, and if they haven't been set, falls back to
// using UnixGetDeviceAttributes() to directly query an existing device file.
-func instanceUnixGetDeviceAttributes(devicesPath string, prefix string, config config.Device) (string, int, int, error) {
+func unixDeviceInstanceAttributes(devicesPath string, prefix string, config config.Device) (string, int, int, error) {
// Check if we've been passed major and minor numbers already.
var tmp int
var err error
@@ -56,7 +57,7 @@ func instanceUnixGetDeviceAttributes(devicesPath string, prefix string, config c
devPath := filepath.Join(devicesPath, devName)
if dType == "" || dMajor < 0 || dMinor < 0 {
- dType, dMajor, dMinor, err = UnixGetDeviceAttributes(devPath)
+ dType, dMajor, dMinor, err = UnixDeviceAttributes(devPath)
if err != nil {
return dType, dMajor, dMinor, err
}
@@ -65,8 +66,8 @@ func instanceUnixGetDeviceAttributes(devicesPath string, prefix string, config c
return dType, dMajor, dMinor, err
}
-// UnixGetDeviceAttributes returns the decice type, major and minor numbers for a device.
-func UnixGetDeviceAttributes(path string) (string, int, int, error) {
+// UnixDeviceAttributes returns the decice type, major and minor numbers for a device.
+func UnixDeviceAttributes(path string) (string, int, int, error) {
// Get a stat struct from the provided path
stat := unix.Stat_t{}
err := unix.Stat(path, &stat)
@@ -90,7 +91,7 @@ func UnixGetDeviceAttributes(path string) (string, int, int, error) {
return dType, major, minor, nil
}
-func unixGetDeviceModeOct(strmode string) (int, error) {
+func unixDeviceModeOct(strmode string) (int, error) {
// Default mode
if strmode == "" {
return 0600, nil
@@ -105,12 +106,43 @@ func unixGetDeviceModeOct(strmode string) (int, error) {
return int(i), nil
}
-// UnixCreateDevice Unix devices handling.
-func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath string, prefix string, m config.Device, defaultMode bool) ([]string, error) {
+// UnixDevice contains information about a created UNIX device.
+type UnixDevice struct {
+ HostPath string // Absolute path to the device on the host.
+ RelativePath string // Relative path where the device will be mounted inside instance.
+ Type string // Type of device; c (for char) or b for (block).
+ Major int // Major number.
+ Minor int // Minor number.
+ Mode os.FileMode // File mode.
+ UID int // Owner UID.
+ GID int // Owner GID.
+}
+
+// unixDeviceDestPath returns the absolute path for a device inside an instance.
+// This is based on the "path" property of the devices' config, or the "source" property if "path"
+// not defined.
+func unixDeviceDestPath(m config.Device) string {
+ destPath := m["path"]
+ if destPath == "" {
+ destPath = m["source"]
+ }
+
+ return destPath
+}
+
+// UnixDeviceCreate creates a UNIX device (either block or char). If the supplied device config map
+// contains a major and minor number for the device, then a stat is avoided, otherwise this info
+// retrieved from the origin device. Similarly, if a mode is supplied in the device config map or
+// defaultMode is set as true, then the device is created with the supplied or default mode (0660)
+// respectively, otherwise the origin device's mode is used. If the device config doesn't contain a
+// type field then it defaults to created a unix-char device. The ownership of the created device
+// defaults to root (0) but can be specified with the uid and gid fields in the device config map.
+// It returns a UnixDevice containing information about the device created.
+func UnixDeviceCreate(s *state.State, idmapSet *idmap.IdmapSet, devicesPath string, prefix string, m config.Device, defaultMode bool) (*UnixDevice, error) {
var err error
- var major, minor int
+ d := UnixDevice{}
- // Extra checks for nesting
+ // Extra checks for nesting.
if s.OS.RunningInUserNS {
for key, value := range m {
if shared.StringInSlice(key, []string{"major", "minor", "mode", "uid", "gid"}) && value != "" {
@@ -125,71 +157,70 @@ func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath stri
}
srcPath = shared.HostPath(srcPath)
- // Get the major/minor of the device we want to create
+ // Get the major/minor of the device we want to create.
if m["major"] == "" && m["minor"] == "" {
- // If no major and minor are set, use those from the device on the host
- _, major, minor, err = UnixGetDeviceAttributes(srcPath)
+ // If no major and minor are set, use those from the device on the host.
+ _, d.Major, d.Minor, err = UnixDeviceAttributes(srcPath)
if err != nil {
return nil, fmt.Errorf("Failed to get device attributes for %s: %s", m["path"], err)
}
} else if m["major"] == "" || m["minor"] == "" {
return nil, fmt.Errorf("Both major and minor must be supplied for device: %s", m["path"])
} else {
- major, err = strconv.Atoi(m["major"])
+ d.Major, err = strconv.Atoi(m["major"])
if err != nil {
return nil, fmt.Errorf("Bad major %s in device %s", m["major"], m["path"])
}
- minor, err = strconv.Atoi(m["minor"])
+ d.Minor, err = strconv.Atoi(m["minor"])
if err != nil {
return nil, fmt.Errorf("Bad minor %s in device %s", m["minor"], m["path"])
}
}
- // Get the device mode
- mode := os.FileMode(0660)
+ // Get the device mode (defaults to 0660 if not supplied).
+ d.Mode = os.FileMode(0660)
if m["mode"] != "" {
- tmp, err := unixGetDeviceModeOct(m["mode"])
+ tmp, err := unixDeviceModeOct(m["mode"])
if err != nil {
return nil, fmt.Errorf("Bad mode %s in device %s", m["mode"], m["path"])
}
- mode = os.FileMode(tmp)
+ d.Mode = os.FileMode(tmp)
} else if !defaultMode {
- mode, err = shared.GetPathMode(srcPath)
+ d.Mode, err = shared.GetPathMode(srcPath)
if err != nil {
errno, isErrno := shared.GetErrno(err)
if !isErrno || errno != unix.ENOENT {
return nil, fmt.Errorf("Failed to retrieve mode of device %s: %s", m["path"], err)
}
- mode = os.FileMode(0660)
+ d.Mode = os.FileMode(0660)
}
}
if m["type"] == "unix-block" {
- mode |= unix.S_IFBLK
+ d.Mode |= unix.S_IFBLK
+ d.Type = "b"
} else {
- mode |= unix.S_IFCHR
+ d.Mode |= unix.S_IFCHR
+ d.Type = "c"
}
- // Get the device owner
- uid := 0
- gid := 0
-
+ // Get the device owner.
if m["uid"] != "" {
- uid, err = strconv.Atoi(m["uid"])
+ d.UID, err = strconv.Atoi(m["uid"])
if err != nil {
return nil, fmt.Errorf("Invalid uid %s in device %s", m["uid"], m["path"])
}
}
if m["gid"] != "" {
- gid, err = strconv.Atoi(m["gid"])
+ d.GID, err = strconv.Atoi(m["gid"])
if err != nil {
return nil, fmt.Errorf("Invalid gid %s in device %s", m["gid"], m["path"])
}
}
- // Create the devices directory if missing
+ // Create the devices directory if missing.
if !shared.PathExists(devicesPath) {
os.Mkdir(devicesPath, 0711)
if err != nil {
@@ -197,29 +228,26 @@ func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath stri
}
}
- destPath := m["path"]
- if destPath == "" {
- destPath = m["source"]
- }
+ destPath := unixDeviceDestPath(m)
relativeDestPath := strings.TrimPrefix(destPath, "/")
- devName := fmt.Sprintf("%s.%s", strings.Replace(prefix, "/", "-", -1), strings.Replace(relativeDestPath, "/", "-", -1))
+ devName := unixDeviceEncode(unixDeviceJoinPath(prefix, relativeDestPath))
devPath := filepath.Join(devicesPath, devName)
- // Create the new entry
+ // Create the new entry.
if !s.OS.RunningInUserNS {
- encodedDeviceNumber := (minor & 0xff) | (major << 8) | ((minor & ^0xff) << 12)
- err := unix.Mknod(devPath, uint32(mode), encodedDeviceNumber)
+ devNum := int(unix.Mkdev(uint32(d.Major), uint32(d.Minor)))
+ err := unix.Mknod(devPath, uint32(d.Mode), devNum)
if err != nil {
return nil, fmt.Errorf("Failed to create device %s for %s: %s", devPath, m["path"], err)
}
- err = os.Chown(devPath, uid, gid)
+ err = os.Chown(devPath, d.UID, d.GID)
if err != nil {
return nil, fmt.Errorf("Failed to chown device %s: %s", devPath, err)
}
- // Needed as mknod respects the umask
- err = os.Chmod(devPath, mode)
+ // Needed as mknod respects the umask.
+ err = os.Chmod(devPath, d.Mode)
if err != nil {
return nil, fmt.Errorf("Failed to chmod device %s: %s", devPath, err)
}
@@ -227,7 +255,7 @@ func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath stri
if idmapSet != nil {
err := idmapSet.ShiftFile(devPath)
if err != nil {
- // uidshift failing is weird, but not a big problem. Log and proceed
+ // uidshift failing is weird, but not a big problem. Log and proceed.
logger.Debugf("Failed to uidshift device %s: %s\n", m["path"], err)
}
}
@@ -244,5 +272,259 @@ func UnixCreateDevice(s *state.State, idmapSet *idmap.IdmapSet, devicesPath stri
}
}
- return []string{devPath, relativeDestPath}, nil
+ d.HostPath = devPath
+ d.RelativePath = relativeDestPath
+ return &d, nil
+}
+
+// unixDeviceSetup creates a UNIX device on host and then configures supplied RunConfig with the
+// mount and cgroup rule instructions to have it be attached to the instance. If defaultMode is true
+// or mode is supplied in the device config then the origin device does not need to be accessed for
+// its file mode.
+func unixDeviceSetup(s *state.State, devicesPath string, typePrefix string, deviceName string, m config.Device, defaultMode bool, runConf *RunConfig) error {
+ // Before creating the device, check that another existing device isn't using the same mount
+ // path inside the instance as our device. If we find an existing device with the same mount
+ // path we will skip mounting our device inside the instance. This can happen when multiple
+ // LXD devices share the same parent device (such as Nvidia GPUs and Infiniband devices).
+
+ // Convert the requested dest path inside the instance to an encoded relative one.
+ ourDestPath := unixDeviceDestPath(m)
+ ourEncRelDestFile := unixDeviceEncode(strings.TrimPrefix(ourDestPath, "/"))
+
+ // Load all existing host devices.
+ dents, err := ioutil.ReadDir(devicesPath)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return err
+ }
+ }
+
+ dupe := false
+ for _, ent := range dents {
+ devName := ent.Name()
+
+ // Remove the LXD device type and name prefix, leaving just the encoded dest path.
+ idx := strings.LastIndex(devName, ".")
+ if idx == -1 {
+ return fmt.Errorf("Invalid device name \"%s\"", devName)
+ }
+
+ encRelDestFile := devName[idx+1:]
+
+ // If the encoded relative path of the device file matches the encoded relative dest
+ // path of our new device then return as we do not want to instruct LXD to mount
+ // the device and create cgroup rules.
+ if encRelDestFile == ourEncRelDestFile {
+ dupe = true // There is an existing device using the same mount path.
+ break
+ }
+ }
+
+ // Create the device on the host.
+ ourPrefix := unixDeviceEncode(unixDeviceJoinPath(typePrefix, deviceName))
+ d, err := UnixDeviceCreate(s, nil, devicesPath, ourPrefix, m, defaultMode)
+ if err != nil {
+ return err
+ }
+
+ // If there was an existing device using the same mount path detected then skip mounting.
+ if dupe {
+ return nil
+ }
+
+ // Instruct liblxc to perform the mount.
+ runConf.Mounts = append(runConf.Mounts, MountEntryItem{
+ DevPath: d.HostPath,
+ TargetPath: d.RelativePath,
+ FSType: "none",
+ Opts: []string{"bind", "create=file"},
+ })
+
+ // Instruct liblxc to setup the cgroup rule.
+ runConf.CGroups = append(runConf.CGroups, RunConfigItem{
+ Key: "devices.allow",
+ Value: fmt.Sprintf("%s %d:%d rwm", d.Type, d.Major, d.Minor),
+ })
+
+ return nil
+}
+
+// unixDeviceSetupCharNum calls unixDeviceSetup and overrides the supplied device config with the
+// type as "unix-char" and the supplied major and minor numbers. This function can be used when you
+// already know the device's major and minor numbers to avoid unixDeviceSetup() having to stat the
+// device to ascertain these attributes. If defaultMode is true or mode is supplied in the device
+// config then the origin device does not need to be accessed for its file mode.
+func unixDeviceSetupCharNum(s *state.State, devicesPath string, typePrefix string, deviceName string, m config.Device, major int, minor int, path string, defaultMode bool, runConf *RunConfig) error {
+ configCopy := config.Device{}
+ for k, v := range m {
+ configCopy[k] = v
+ }
+
+ // Overridng these in the config copy should avoid the need for unixDeviceSetup to stat
+ // the origin device to ascertain this information.
+ configCopy["type"] = "unix-char"
+ configCopy["major"] = fmt.Sprintf("%d", major)
+ configCopy["minor"] = fmt.Sprintf("%d", minor)
+ configCopy["path"] = path
+
+ return unixDeviceSetup(s, devicesPath, typePrefix, deviceName, configCopy, defaultMode, runConf)
+}
+
+// UnixDeviceExists checks if the unix device already exists in devices path.
+func UnixDeviceExists(devicesPath string, prefix string, path string) bool {
+ relativeDestPath := strings.TrimPrefix(path, "/")
+ devName := fmt.Sprintf("%s.%s", unixDeviceEncode(prefix), unixDeviceEncode(relativeDestPath))
+ devPath := filepath.Join(devicesPath, devName)
+
+ return shared.PathExists(devPath)
+}
+
+// unixDeviceEncode encodes a string to be used as part of a file name in the LXD devices path.
+// The encoding scheme replaces "-" with "--" and then "/" with "-".
+func unixDeviceEncode(text string) string {
+ return strings.Replace(strings.Replace(text, "-", "--", -1), "/", "-", -1)
+}
+
+// unixDeviceDecode decodes a string used in the LXD devices path back to its original form.
+// The decoding scheme converts "-" back to "/" and "--" back to "-".
+func unixDeviceDecode(text string) string {
+ // This converts "--" to the null character "\0" first, to allow remaining "-" chars to be
+ // converted back to "/" before making a final pass to convert "\0" back to original "-".
+ return strings.Replace(strings.Replace(strings.Replace(text, "--", "\000", -1), "-", "/", -1), "\000", "-", -1)
+}
+
+// unixDeviceJoinPath joins together prefix and text delimited by a "." for device path generation.
+func unixDeviceJoinPath(prefix string, text string) string {
+ return fmt.Sprintf("%s.%s", prefix, text)
+}
+
+// unixRemoveDevice identifies all files related to the supplied typePrefix and deviceName and then
+// populates the supplied runConf with the instructions to remove cgroup rules and unmount devices.
+// It detects if any other devices attached to the instance that share the same prefix have the same
+// relative mount path inside the instance encoded into the file name. If there is another device
+// that shares the same mount path then the unmount rule is not added to the runConf as the device
+// may still be in use with another LXD device.
+func unixDeviceRemove(devicesPath string, typePrefix string, deviceName string, runConf *RunConfig) error {
+ // Load all devices.
+ dents, err := ioutil.ReadDir(devicesPath)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return err
+ }
+ }
+
+ ourPrefix := unixDeviceEncode(unixDeviceJoinPath(typePrefix, deviceName))
+ ourDevs := []string{}
+ otherDevs := []string{}
+
+ for _, ent := range dents {
+ devName := ent.Name()
+
+ // This device file belongs our LXD device.
+ if strings.HasPrefix(devName, ourPrefix) {
+ ourDevs = append(ourDevs, devName)
+ continue
+ }
+
+ // This device file belongs to another LXD device.
+ otherDevs = append(otherDevs, devName)
+ }
+
+ // It is possible for some LXD devices to share the same device on the same mount point
+ // inside the instance. We extract the relative path of the device that is encoded into its
+ // name on the host so that we can compare the device files for our own device and check
+ // none of them use the same mount point.
+ encRelDevFiles := []string{}
+ for _, otherDev := range otherDevs {
+ // Remove the LXD device type and name prefix, leaving just the encoded dest path.
+ idx := strings.LastIndex(otherDev, ".")
+ if idx == -1 {
+ return fmt.Errorf("Invalid device name \"%s\"", otherDev)
+ }
+
+ encRelDestFile := otherDev[idx+1:]
+ encRelDevFiles = append(encRelDevFiles, encRelDestFile)
+ }
+
+ // Check that none of our devices are in use by another LXD device.
+ for _, ourDev := range ourDevs {
+ // Remove the LXD device type and name prefix, leaving just the encoded dest path.
+ idx := strings.LastIndex(ourDev, ".")
+ if idx == -1 {
+ return fmt.Errorf("Invalid device name \"%s\"", ourDev)
+ }
+
+ ourEncRelDestFile := ourDev[idx+1:]
+
+ // Look for devices for other LXD devices that match the same path.
+ dupe := false
+ for _, encRelDevFile := range encRelDevFiles {
+ if encRelDevFile == ourEncRelDestFile {
+ dupe = true
+ break
+ }
+ }
+
+ // If a device has been found that points to the same device inside the instance
+ // then we cannot request it be umounted inside the instance as it's still in use.
+ if dupe {
+ continue
+ }
+
+ // Append this device to the mount rules (these will be unmounted).
+ runConf.Mounts = append(runConf.Mounts, MountEntryItem{
+ TargetPath: unixDeviceDecode(ourEncRelDestFile),
+ })
+
+ absDevPath := filepath.Join(devicesPath, ourDev)
+ dType, dMajor, dMinor, err := UnixDeviceAttributes(absDevPath)
+ if err != nil {
+ return fmt.Errorf("Failed to get UNIX device attributes for '%s': %v", absDevPath, err)
+ }
+
+ // Append a deny cgroup fule for this device.
+ runConf.CGroups = append(runConf.CGroups, RunConfigItem{
+ Key: "devices.deny",
+ Value: fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor),
+ })
+ }
+
+ return nil
+}
+
+// unixDeviceDeleteFiles removes all host side device files for a particular LXD device.
+// This should be run after the files have been detached from the instance using unixDeviceRemove().
+func unixDeviceDeleteFiles(s *state.State, devicesPath string, typePrefix string, deviceName string) error {
+ ourPrefix := unixDeviceEncode(unixDeviceJoinPath(typePrefix, deviceName))
+
+ // Load all devices.
+ dents, err := ioutil.ReadDir(devicesPath)
+ if err != nil {
+ if !os.IsNotExist(err) {
+ return err
+ }
+ }
+
+ // Remove our host side device files.
+ for _, ent := range dents {
+ devName := ent.Name()
+
+ // This device file belongs our LXD device.
+ if strings.HasPrefix(devName, ourPrefix) {
+ devPath := filepath.Join(devicesPath, devName)
+
+ // Remove the host side mount.
+ if s.OS.RunningInUserNS {
+ unix.Unmount(devPath, unix.MNT_DETACH)
+ }
+
+ // Remove the host side device file.
+ err = os.Remove(devPath)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
}
From 0789df4b15357dfa5d1670c6dc0e2e4068762368 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 11:43:39 +0100
Subject: [PATCH 07/26] container/lxc: Updates gpu device support to use device
package
Also updates use of device.UnixCreateDevice() to use the new UnixDevice struct returned.
Removes gpu related code that has moved into device package.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container_lxc.go | 257 +++++--------------------------------------
1 file changed, 27 insertions(+), 230 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 25e6789648..3d25178c76 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -510,7 +510,7 @@ func containerLXCCreate(s *state.State, args db.ContainerArgs) (container, error
err = c.deviceAdd(k, m)
if err != nil && err != device.ErrUnsupportedDevType {
c.Delete()
- return nil, err
+ return nil, errors.Wrapf(err, "Failed to add device '%s'", k)
}
}
}
@@ -1950,6 +1950,8 @@ func (c *containerLXC) deviceStart(deviceName string, rawConfig map[string]strin
}
}
+ // If running, run post start hooks now (if not running LXD will run them
+ // once the instance is started).
err = c.runHooks(runConfig.PostHooks)
if err != nil {
return nil, err
@@ -2110,6 +2112,7 @@ func (c *containerLXC) deviceStop(deviceName string, rawConfig map[string]string
}
}
+ // Run post stop hooks irrespective of run state of instance.
err = c.runHooks(runConfig.PostHooks)
if err != nil {
return err
@@ -2307,7 +2310,7 @@ func (c *containerLXC) setupUnixDevice(prefix string, dev config.Device, major i
return err
}
- paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, temp, defaultMode)
+ d, err := device.UnixDeviceCreate(c.state, idmapSet, c.DevicesPath(), prefix, temp, defaultMode)
if err != nil {
logger.Debug("Failed to create device", log.Ctx{"err": err, "device": prefix})
if createMustSucceed {
@@ -2317,8 +2320,8 @@ func (c *containerLXC) setupUnixDevice(prefix string, dev config.Device, major i
return nil
}
- devPath := shared.EscapePathFstab(paths[0])
- tgtPath := shared.EscapePathFstab(paths[1])
+ devPath := shared.EscapePathFstab(d.HostPath)
+ tgtPath := shared.EscapePathFstab(d.RelativePath)
val := fmt.Sprintf("%s %s none bind,create=file 0 0", devPath, tgtPath)
return lxcSetConfigItem(c.c, "lxc.mount.entry", val)
@@ -2559,7 +2562,7 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
}
// Unix device
- paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), fmt.Sprintf("unix.%s", k), m, true)
+ d, err := device.UnixDeviceCreate(c.state, idmapSet, c.DevicesPath(), fmt.Sprintf("unix.%s", k), m, true)
if err != nil {
// Deal with device hotplug
if m["required"] == "" || shared.IsTrue(m["required"]) {
@@ -2579,10 +2582,10 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
}
continue
}
- devPath := paths[0]
+ devPath := d.HostPath
if c.isCurrentlyPrivileged() && !c.state.OS.RunningInUserNS && c.state.OS.CGroupDevicesController {
// Add the new device cgroup rule
- dType, dMajor, dMinor, err := device.UnixGetDeviceAttributes(devPath)
+ dType, dMajor, dMinor, err := device.UnixDeviceAttributes(devPath)
if err != nil {
if m["required"] == "" || shared.IsTrue(m["required"]) {
return "", postStartHooks, err
@@ -2612,67 +2615,6 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
return "", postStartHooks, err
}
}
- } else if m["type"] == "gpu" {
- allGpus := deviceWantsAllGPUs(m)
- gpus, nvidiaDevices, err := deviceLoadGpu(allGpus)
- if err != nil {
- return "", postStartHooks, err
- }
-
- sawNvidia := false
- found := false
- for _, gpu := range gpus {
- if (m["vendorid"] != "" && gpu.vendorID != m["vendorid"]) ||
- (m["pci"] != "" && gpu.pci != m["pci"]) ||
- (m["productid"] != "" && gpu.productID != m["productid"]) ||
- (m["id"] != "" && gpu.id != m["id"]) {
- continue
- }
-
- found = true
-
- err := c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, true, false)
- if err != nil {
- return "", postStartHooks, err
- }
-
- if !gpu.isNvidia {
- continue
- }
-
- if gpu.nvidia.path != "" {
- err = c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path, true, false)
- if err != nil {
- return "", postStartHooks, err
- }
- } else if !allGpus {
- errMsg := fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
- logger.Errorf("%s", errMsg)
- return "", postStartHooks, errMsg
- }
-
- sawNvidia = true
- }
-
- if sawNvidia {
- for _, gpu := range nvidiaDevices {
- if shared.IsTrue(c.expandedConfig["nvidia.runtime"]) {
- if !gpu.isCard {
- continue
- }
- }
- err := c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, true, false)
- if err != nil {
- return "", postStartHooks, err
- }
- }
- }
-
- if !found {
- msg := "Failed to detect requested GPU device"
- logger.Error(msg)
- return "", postStartHooks, fmt.Errorf(msg)
- }
} else if m["type"] == "disk" {
if m["path"] != "/" {
diskDevices[k] = m
@@ -2682,7 +2624,7 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
runConfig, err := c.deviceStart(k, m, false)
if err != device.ErrUnsupportedDevType {
if err != nil {
- return "", postStartHooks, err
+ return "", postStartHooks, errors.Wrapf(err, "Failed to start device '%s'", k)
}
// Pass any cgroups rules into LXC.
@@ -3392,7 +3334,7 @@ func (c *containerLXC) cleanupDevices(netns string) {
if err == device.ErrUnsupportedDevType {
continue
} else if err != nil {
- logger.Errorf("Failed to stop device: %v", err)
+ logger.Errorf("Failed to stop device '%s': %v", k, err)
}
}
}
@@ -3994,7 +3936,7 @@ func (c *containerLXC) Delete() error {
for k, m := range c.expandedDevices {
err = c.deviceRemove(k, m)
if err != nil && err != device.ErrUnsupportedDevType {
- return err
+ return errors.Wrapf(err, "Failed to remove device '%s'", k)
}
}
}
@@ -5007,7 +4949,7 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
destPath = m["source"]
}
- if !c.deviceExistsInDevicesFolder(prefix, destPath) && (m["required"] != "" && !shared.IsTrue(m["required"])) {
+ if !device.UnixDeviceExists(c.DevicesPath(), prefix, destPath) && (m["required"] != "" && !shared.IsTrue(m["required"])) {
continue
}
@@ -5039,73 +4981,6 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
return err
}
}
- } else if m["type"] == "gpu" {
- allGpus := deviceWantsAllGPUs(m)
- gpus, nvidiaDevices, err := deviceLoadGpu(allGpus)
- if err != nil {
- return err
- }
-
- for _, gpu := range gpus {
- if (m["vendorid"] != "" && gpu.vendorID != m["vendorid"]) ||
- (m["pci"] != "" && gpu.pci != m["pci"]) ||
- (m["productid"] != "" && gpu.productID != m["productid"]) ||
- (m["id"] != "" && gpu.id != m["id"]) {
- continue
- }
-
- err := c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path)
- if err != nil {
- logger.Error("Failed to remove GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
- return err
- }
-
- if !gpu.isNvidia {
- continue
- }
-
- if gpu.nvidia.path != "" {
- err = c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path)
- if err != nil {
- logger.Error("Failed to remove GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
- return err
- }
- } else if !allGpus {
- errMsg := fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
- logger.Errorf("%s", errMsg)
- return errMsg
- }
- }
-
- nvidiaExists := false
- for _, gpu := range gpus {
- if gpu.nvidia.path != "" {
- if c.deviceExistsInDevicesFolder(fmt.Sprintf("unix.%s", k), gpu.path) {
- nvidiaExists = true
- break
- }
- }
- }
-
- if !nvidiaExists {
- for _, gpu := range nvidiaDevices {
- if shared.IsTrue(c.expandedConfig["nvidia.runtime"]) {
- if !gpu.isCard {
- continue
- }
- }
-
- if !c.deviceExistsInDevicesFolder(fmt.Sprintf("unix.%s", k), gpu.path) {
- continue
- }
-
- err = c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path)
- if err != nil {
- logger.Error("Failed to remove GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
- return err
- }
- }
- }
}
}
@@ -5138,75 +5013,6 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
logger.Error("Failed to insert usb device", log.Ctx{"err": err, "usb": usb, "container": c.Name()})
}
}
- } else if m["type"] == "gpu" {
- allGpus := deviceWantsAllGPUs(m)
- gpus, nvidiaDevices, err := deviceLoadGpu(allGpus)
- if err != nil {
- return err
- }
-
- sawNvidia := false
- found := false
- for _, gpu := range gpus {
- if (m["vendorid"] != "" && gpu.vendorID != m["vendorid"]) ||
- (m["pci"] != "" && gpu.pci != m["pci"]) ||
- (m["productid"] != "" && gpu.productID != m["productid"]) ||
- (m["id"] != "" && gpu.id != m["id"]) {
- continue
- }
-
- found = true
-
- err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, false)
- if err != nil {
- logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
- return err
- }
-
- if !gpu.isNvidia {
- continue
- }
-
- if gpu.nvidia.path != "" {
- err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.nvidia.major, gpu.nvidia.minor, gpu.nvidia.path, false)
- if err != nil {
- logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
- return err
- }
- } else if !allGpus {
- errMsg := fmt.Errorf("Failed to detect correct \"/dev/nvidia\" path")
- logger.Errorf("%s", errMsg)
- return errMsg
- }
-
- sawNvidia = true
- }
-
- if sawNvidia {
- for _, gpu := range nvidiaDevices {
- if shared.IsTrue(c.expandedConfig["nvidia.runtime"]) {
- if !gpu.isCard {
- continue
- }
- }
-
- if c.deviceExistsInDevicesFolder(k, gpu.path) {
- continue
- }
-
- err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, gpu.major, gpu.minor, gpu.path, false)
- if err != nil {
- logger.Error("Failed to insert GPU device", log.Ctx{"err": err, "gpu": gpu, "container": c.Name()})
- return err
- }
- }
- }
-
- if !found {
- msg := "Failed to detect requested GPU device"
- logger.Error(msg)
- return fmt.Errorf(msg)
- }
}
}
@@ -5437,13 +5243,13 @@ func (c *containerLXC) updateDevices(removeDevices map[string]config.Device, add
if err == device.ErrUnsupportedDevType {
continue // No point in trying to remove device below.
} else if err != nil {
- return err
+ return errors.Wrapf(err, "Failed to stop device '%s'", k)
}
}
err := c.deviceRemove(k, m)
if err != nil && err != device.ErrUnsupportedDevType {
- return err
+ return errors.Wrapf(err, "Failed to remove device '%s'", k)
}
}
@@ -5452,13 +5258,13 @@ func (c *containerLXC) updateDevices(removeDevices map[string]config.Device, add
if err == device.ErrUnsupportedDevType {
continue // No point in trying to start device below.
} else if err != nil {
- return err
+ return errors.Wrapf(err, "Failed to add device '%s'", k)
}
if isRunning {
_, err := c.deviceStart(k, m, isRunning)
if err != nil && err != device.ErrUnsupportedDevType {
- return err
+ return errors.Wrapf(err, "Failed to start device '%s'", k)
}
}
}
@@ -5466,7 +5272,7 @@ func (c *containerLXC) updateDevices(removeDevices map[string]config.Device, add
for k, m := range updateDevices {
err := c.deviceUpdate(k, m, oldExpandedDevices[k], isRunning)
if err != nil && err != device.ErrUnsupportedDevType {
- return err
+ return errors.Wrapf(err, "Failed to update device '%s'", k)
}
}
@@ -6949,15 +6755,6 @@ func (c *containerLXC) removeMount(mount string) error {
return nil
}
-// Check if the unix device already exists.
-func (c *containerLXC) deviceExistsInDevicesFolder(prefix string, path string) bool {
- relativeDestPath := strings.TrimPrefix(path, "/")
- devName := fmt.Sprintf("%s.%s", strings.Replace(prefix, "/", "-", -1), strings.Replace(relativeDestPath, "/", "-", -1))
- devPath := filepath.Join(c.DevicesPath(), devName)
-
- return shared.PathExists(devPath)
-}
-
func (c *containerLXC) insertUnixDevice(prefix string, m config.Device, defaultMode bool) error {
// Check that the container is running
if !c.IsRunning() {
@@ -6970,12 +6767,12 @@ func (c *containerLXC) insertUnixDevice(prefix string, m config.Device, defaultM
}
// Create the device on the host
- paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, m, defaultMode)
+ d, err := device.UnixDeviceCreate(c.state, idmapSet, c.DevicesPath(), prefix, m, defaultMode)
if err != nil {
return fmt.Errorf("Failed to setup device: %s", err)
}
- devPath := paths[0]
- tgtPath := paths[1]
+ devPath := d.HostPath
+ tgtPath := d.RelativePath
// Bind-mount it into the container
err = c.insertMount(devPath, tgtPath, "none", unix.MS_BIND, false)
@@ -7009,7 +6806,7 @@ func (c *containerLXC) insertUnixDevice(prefix string, m config.Device, defaultM
}
if dType == "" || dMajor < 0 || dMinor < 0 {
- dType, dMajor, dMinor, err = device.UnixGetDeviceAttributes(devPath)
+ dType, dMajor, dMinor, err = device.UnixDeviceAttributes(devPath)
if err != nil {
return err
}
@@ -7068,12 +6865,12 @@ func (c *containerLXC) InsertSeccompUnixDevice(prefix string, m config.Device, p
return err
}
- paths, err := device.UnixCreateDevice(c.state, idmapSet, c.DevicesPath(), prefix, m, true)
+ d, err := device.UnixDeviceCreate(c.state, idmapSet, c.DevicesPath(), prefix, m, true)
if err != nil {
return fmt.Errorf("Failed to setup device: %s", err)
}
- devPath := paths[0]
- tgtPath := paths[1]
+ devPath := d.HostPath
+ tgtPath := d.RelativePath
// Bind-mount it into the container
defer os.Remove(devPath)
@@ -7140,7 +6937,7 @@ func (c *containerLXC) removeUnixDevice(prefix string, m config.Device, eject bo
devPath := filepath.Join(c.DevicesPath(), devName)
if dType == "" || dMajor < 0 || dMinor < 0 {
- dType, dMajor, dMinor, err = device.UnixGetDeviceAttributes(devPath)
+ dType, dMajor, dMinor, err = device.UnixDeviceAttributes(devPath)
if err != nil {
return err
}
From e73b5eea1da29d4cda25c4a4cdab6e4da40a1c0f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Thu, 8 Aug 2019 17:49:11 +0100
Subject: [PATCH 08/26] device/infiniband/sriov: Updates use of unix device
functions
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/infiniband_sriov.go | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/lxd/device/infiniband_sriov.go b/lxd/device/infiniband_sriov.go
index 831bc5a753..7a55588317 100644
--- a/lxd/device/infiniband_sriov.go
+++ b/lxd/device/infiniband_sriov.go
@@ -199,7 +199,7 @@ func (d *infinibandSRIOV) Stop() (*RunConfig, error) {
NetworkInterface: []RunConfigItem{{Key: "link", Value: v["host_name"]}},
}
- err := infinibandRemoveDevices(d.instance.DevicesPath(), d.name, &runConf)
+ err := unixDeviceRemove(d.instance.DevicesPath(), IBDevPrefix, d.name, &runConf)
if err != nil {
return nil, err
}
@@ -216,12 +216,12 @@ func (d *infinibandSRIOV) postStop() error {
})
// Remove infiniband host files for this device.
- err := infinibandDeleteHostFiles(d.state, d.instance.DevicesPath(), d.name)
+ err := unixDeviceDeleteFiles(d.state, d.instance.DevicesPath(), IBDevPrefix, d.name)
if err != nil {
- return err
+ return fmt.Errorf("Failed to delete files for device '%s': %v", d.name, err)
}
- // Restpre hwaddr and mtu.
+ // Restore hwaddr and mtu.
v := d.volatileGet()
if v["host_name"] != "" {
err := networkRestorePhysicalNic(v["host_name"], v)
From e7148e4bad5fc4fcf22ed722719e8ca25fdb2ed4 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 9 Aug 2019 10:53:20 +0100
Subject: [PATCH 09/26] device/infiniband/physical: Updates use of unix device
functions
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/infiniband_physical.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lxd/device/infiniband_physical.go b/lxd/device/infiniband_physical.go
index ab2d407702..2acba770c4 100644
--- a/lxd/device/infiniband_physical.go
+++ b/lxd/device/infiniband_physical.go
@@ -120,7 +120,7 @@ func (d *infinibandPhysical) Stop() (*RunConfig, error) {
},
}
- err := infinibandRemoveDevices(d.instance.DevicesPath(), d.name, &runConf)
+ err := unixDeviceRemove(d.instance.DevicesPath(), IBDevPrefix, d.name, &runConf)
if err != nil {
return nil, err
}
@@ -137,9 +137,9 @@ func (d *infinibandPhysical) postStop() error {
})
// Remove infiniband host files for this device.
- err := infinibandDeleteHostFiles(d.state, d.instance.DevicesPath(), d.name)
+ err := unixDeviceDeleteFiles(d.state, d.instance.DevicesPath(), IBDevPrefix, d.name)
if err != nil {
- return err
+ return fmt.Errorf("Failed to delete files for device '%s': %v", d.name, err)
}
// Restpre hwaddr and mtu.
From d754428764fb65cc92a89efefbb505689fa5543b Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 10:29:59 +0100
Subject: [PATCH 10/26] container/lxc: Updates all device major/minor parsing
to use uint32
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container_lxc.go | 46 +++++++++++++++++++++++---------------------
1 file changed, 24 insertions(+), 22 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 3d25178c76..6a97f78242 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -6781,21 +6781,21 @@ func (c *containerLXC) insertUnixDevice(prefix string, m config.Device, defaultM
}
// Check if we've been passed major and minor numbers already.
- var tmp int
- dMajor := -1
+ var dMajor, dMinor uint32
if m["major"] != "" {
- tmp, err = strconv.Atoi(m["major"])
- if err == nil {
- dMajor = tmp
+ tmp, err := strconv.ParseUint(m["major"], 10, 32)
+ if err != nil {
+ return err
}
+ dMajor = uint32(tmp)
}
- dMinor := -1
if m["minor"] != "" {
- tmp, err = strconv.Atoi(m["minor"])
- if err == nil {
- dMinor = tmp
+ tmp, err := strconv.ParseUint(m["minor"], 10, 32)
+ if err != nil {
+ return err
}
+ dMinor = uint32(tmp)
}
dType := ""
@@ -6805,7 +6805,7 @@ func (c *containerLXC) insertUnixDevice(prefix string, m config.Device, defaultM
dType = "b"
}
- if dType == "" || dMajor < 0 || dMinor < 0 {
+ if dType == "" || m["major"] == "" || m["minor"] == "" {
dType, dMajor, dMinor, err = device.UnixDeviceAttributes(devPath)
if err != nil {
return err
@@ -6902,22 +6902,24 @@ func (c *containerLXC) removeUnixDevice(prefix string, m config.Device, eject bo
}
// Check if we've been passed major and minor numbers already.
- var tmp int
var err error
- dMajor := -1
+ var dMajor, dMinor uint32
if m["major"] != "" {
- tmp, err = strconv.Atoi(m["major"])
- if err == nil {
- dMajor = tmp
+ tmp, err := strconv.ParseUint(m["major"], 10, 32)
+ if err != nil {
+ return err
}
+
+ dMajor = uint32(tmp)
}
- dMinor := -1
if m["minor"] != "" {
- tmp, err = strconv.Atoi(m["minor"])
- if err == nil {
- dMinor = tmp
+ tmp, err := strconv.ParseUint(m["minor"], 10, 32)
+ if err != nil {
+ return err
}
+
+ dMinor = uint32(tmp)
}
dType := ""
@@ -6936,7 +6938,7 @@ func (c *containerLXC) removeUnixDevice(prefix string, m config.Device, eject bo
devName := fmt.Sprintf("%s.%s", strings.Replace(prefix, "/", "-", -1), strings.Replace(relativeDestPath, "/", "-", -1))
devPath := filepath.Join(c.DevicesPath(), devName)
- if dType == "" || dMajor < 0 || dMinor < 0 {
+ if dType == "" || m["major"] == "" || m["minor"] == "" {
dType, dMajor, dMinor, err = device.UnixDeviceAttributes(devPath)
if err != nil {
return err
@@ -6945,14 +6947,14 @@ func (c *containerLXC) removeUnixDevice(prefix string, m config.Device, eject bo
if c.isCurrentlyPrivileged() && !c.state.OS.RunningInUserNS && c.state.OS.CGroupDevicesController {
// Remove the device cgroup rule
- err = c.CGroupSet("devices.deny", fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor))
+ err := c.CGroupSet("devices.deny", fmt.Sprintf("%s %d:%d rwm", dType, dMajor, dMinor))
if err != nil {
return err
}
}
if eject && c.FileExists(relativeDestPath) == nil {
- err = c.removeMount(destPath)
+ err := c.removeMount(destPath)
if err != nil {
return fmt.Errorf("Error unmounting the device: %s", err)
}
From 5a0e0c14c6c78c93afad9d890dbd70c55191162b Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 10:30:58 +0100
Subject: [PATCH 11/26] device/device/utils/unix: Updates all device
major/minor parsing to use uin32
Also moves away from shared.Major/Minor functions to unix package.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_unix.go | 44 ++++++++++++++++++---------------
1 file changed, 24 insertions(+), 20 deletions(-)
diff --git a/lxd/device/device_utils_unix.go b/lxd/device/device_utils_unix.go
index 67e96a44fd..b51e870734 100644
--- a/lxd/device/device_utils_unix.go
+++ b/lxd/device/device_utils_unix.go
@@ -20,24 +20,25 @@ import (
// unixDeviceInstanceAttributes returns the UNIX device attributes for an instance device.
// Uses supplied device config for device properties, and if they haven't been set, falls back to
// using UnixGetDeviceAttributes() to directly query an existing device file.
-func unixDeviceInstanceAttributes(devicesPath string, prefix string, config config.Device) (string, int, int, error) {
+func unixDeviceInstanceAttributes(devicesPath string, prefix string, config config.Device) (string, uint32, uint32, error) {
// Check if we've been passed major and minor numbers already.
- var tmp int
var err error
- dMajor := -1
+ var dMajor, dMinor uint32
+
if config["major"] != "" {
- tmp, err = strconv.Atoi(config["major"])
- if err == nil {
- dMajor = tmp
+ tmp, err := strconv.ParseUint(config["major"], 10, 32)
+ if err != nil {
+ return "", 0, 0, err
}
+ dMajor = uint32(tmp)
}
- dMinor := -1
if config["minor"] != "" {
- tmp, err = strconv.Atoi(config["minor"])
- if err == nil {
- dMinor = tmp
+ tmp, err := strconv.ParseUint(config["minor"], 10, 32)
+ if err != nil {
+ return "", 0, 0, err
}
+ dMinor = uint32(tmp)
}
dType := ""
@@ -56,7 +57,8 @@ func unixDeviceInstanceAttributes(devicesPath string, prefix string, config conf
devName := fmt.Sprintf("%s.%s", strings.Replace(prefix, "/", "-", -1), strings.Replace(relativeDestPath, "/", "-", -1))
devPath := filepath.Join(devicesPath, devName)
- if dType == "" || dMajor < 0 || dMinor < 0 {
+ // If any config options missing then retrieve all the needed set of attributes from device.
+ if dType == "" || config["major"] == "" || config["minor"] == "" {
dType, dMajor, dMinor, err = UnixDeviceAttributes(devPath)
if err != nil {
return dType, dMajor, dMinor, err
@@ -67,7 +69,7 @@ func unixDeviceInstanceAttributes(devicesPath string, prefix string, config conf
}
// UnixDeviceAttributes returns the decice type, major and minor numbers for a device.
-func UnixDeviceAttributes(path string) (string, int, int, error) {
+func UnixDeviceAttributes(path string) (string, uint32, uint32, error) {
// Get a stat struct from the provided path
stat := unix.Stat_t{}
err := unix.Stat(path, &stat)
@@ -86,8 +88,8 @@ func UnixDeviceAttributes(path string) (string, int, int, error) {
}
// Return the device information
- major := shared.Major(stat.Rdev)
- minor := shared.Minor(stat.Rdev)
+ major := unix.Major(stat.Rdev)
+ minor := unix.Minor(stat.Rdev)
return dType, major, minor, nil
}
@@ -111,8 +113,8 @@ type UnixDevice struct {
HostPath string // Absolute path to the device on the host.
RelativePath string // Relative path where the device will be mounted inside instance.
Type string // Type of device; c (for char) or b for (block).
- Major int // Major number.
- Minor int // Minor number.
+ Major uint32 // Major number.
+ Minor uint32 // Minor number.
Mode os.FileMode // File mode.
UID int // Owner UID.
GID int // Owner GID.
@@ -167,15 +169,17 @@ func UnixDeviceCreate(s *state.State, idmapSet *idmap.IdmapSet, devicesPath stri
} else if m["major"] == "" || m["minor"] == "" {
return nil, fmt.Errorf("Both major and minor must be supplied for device: %s", m["path"])
} else {
- d.Major, err = strconv.Atoi(m["major"])
+ tmp, err := strconv.ParseUint(m["major"], 10, 32)
if err != nil {
return nil, fmt.Errorf("Bad major %s in device %s", m["major"], m["path"])
}
+ d.Major = uint32(tmp)
- d.Minor, err = strconv.Atoi(m["minor"])
+ tmp, err = strconv.ParseUint(m["minor"], 10, 32)
if err != nil {
return nil, fmt.Errorf("Bad minor %s in device %s", m["minor"], m["path"])
}
+ d.Minor = uint32(tmp)
}
// Get the device mode (defaults to 0660 if not supplied).
@@ -235,7 +239,7 @@ func UnixDeviceCreate(s *state.State, idmapSet *idmap.IdmapSet, devicesPath stri
// Create the new entry.
if !s.OS.RunningInUserNS {
- devNum := int(unix.Mkdev(uint32(d.Major), uint32(d.Minor)))
+ devNum := int(unix.Mkdev(d.Major, d.Minor))
err := unix.Mknod(devPath, uint32(d.Mode), devNum)
if err != nil {
return nil, fmt.Errorf("Failed to create device %s for %s: %s", devPath, m["path"], err)
@@ -354,7 +358,7 @@ func unixDeviceSetup(s *state.State, devicesPath string, typePrefix string, devi
// already know the device's major and minor numbers to avoid unixDeviceSetup() having to stat the
// device to ascertain these attributes. If defaultMode is true or mode is supplied in the device
// config then the origin device does not need to be accessed for its file mode.
-func unixDeviceSetupCharNum(s *state.State, devicesPath string, typePrefix string, deviceName string, m config.Device, major int, minor int, path string, defaultMode bool, runConf *RunConfig) error {
+func unixDeviceSetupCharNum(s *state.State, devicesPath string, typePrefix string, deviceName string, m config.Device, major uint32, minor uint32, path string, defaultMode bool, runConf *RunConfig) error {
configCopy := config.Device{}
for k, v := range m {
configCopy[k] = v
From 8fa1e881f7129928b36b9631af44409ed3c18a3f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 10:32:28 +0100
Subject: [PATCH 12/26] device/gpu: Updates all device major/minor parsing to
use uint32
Also moves away from shared.Major/Minor functions to unix package.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/gpu.go | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/lxd/device/gpu.go b/lxd/device/gpu.go
index 3ede69e6c9..b99d7c6603 100644
--- a/lxd/device/gpu.go
+++ b/lxd/device/gpu.go
@@ -30,8 +30,8 @@ type gpuDevice struct {
// DRM node information
id string
path string
- major int
- minor int
+ major uint32
+ minor uint32
// Device information
vendorID string
@@ -60,8 +60,8 @@ func (g *gpuDevice) isNvidiaGpu() bool {
// /dev/nvidia[0-9]+
type nvidiaGpuCard struct {
path string
- major int
- minor int
+ major uint32
+ minor uint32
id string
nvrmVersion string
@@ -76,8 +76,8 @@ type nvidiaGpuCard struct {
type nvidiaGpuDevice struct {
isCard bool
path string
- major int
- minor int
+ major uint32
+ minor uint32
}
// Nvidia container info
@@ -450,18 +450,18 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
continue
}
- majorInt, err := strconv.Atoi(majMinSlice[0])
+ majorInt, err := strconv.ParseUint(majMinSlice[0], 10, 32)
if err != nil {
continue
}
- minorInt, err := strconv.Atoi(majMinSlice[1])
+ minorInt, err := strconv.ParseUint(majMinSlice[1], 10, 32)
if err != nil {
continue
}
- tmpGpu.major = majorInt
- tmpGpu.minor = minorInt
+ tmpGpu.major = uint32(majorInt)
+ tmpGpu.minor = uint32(minorInt)
isCard, err := regexp.MatchString("^card[0-9]+", drmEnt.Name())
if err != nil {
@@ -490,9 +490,9 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
}
tmpGpu.nvidia.path = nvidiaPath
- tmpGpu.nvidia.major = shared.Major(stat.Rdev)
- tmpGpu.nvidia.minor = shared.Minor(stat.Rdev)
- tmpGpu.nvidia.id = strconv.Itoa(tmpGpu.nvidia.minor)
+ tmpGpu.nvidia.major = unix.Major(stat.Rdev)
+ tmpGpu.nvidia.minor = unix.Minor(stat.Rdev)
+ tmpGpu.nvidia.id = strconv.FormatInt(int64(tmpGpu.nvidia.minor), 10)
if nvidiaContainer != nil {
tmpGpu.nvidia.nvrmVersion = nvidiaContainer.NVRMVersion
@@ -514,7 +514,7 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
if isCard {
// If it is a card it's minor number will be its id.
- tmpGpu.id = strconv.Itoa(minorInt)
+ tmpGpu.id = strconv.FormatInt(int64(minorInt), 10)
tmp := cardIds{
id: tmpGpu.id,
pci: tmpGpu.pci,
@@ -563,8 +563,8 @@ func (d *gpu) deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevice, error) {
tmpNividiaGpu := nvidiaGpuDevice{
isCard: !validNvidia.MatchString(nvidiaEnt.Name()),
path: nvidiaPath,
- major: shared.Major(stat.Rdev),
- minor: shared.Minor(stat.Rdev),
+ major: unix.Major(stat.Rdev),
+ minor: unix.Minor(stat.Rdev),
}
nvidiaDevices = append(nvidiaDevices, tmpNividiaGpu)
From 2b6b71303caed9bc7f915b1aff7dedeeb79ee979 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 10:48:01 +0100
Subject: [PATCH 13/26] shared/util/linux: Removes Major and Minor functions
These functions are provided by the unix package, and return the major and minor numbers as uint32.
As part of this, GetFileStat has been modified such that it will returned a major and minor number of 0 rather than -1 when the file being requested is not a char or block device.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
shared/util_linux.go | 17 +++--------------
1 file changed, 3 insertions(+), 14 deletions(-)
diff --git a/shared/util_linux.go b/shared/util_linux.go
index 323953b962..f2b8827416 100644
--- a/shared/util_linux.go
+++ b/shared/util_linux.go
@@ -19,16 +19,7 @@ import (
// --- pure Go functions ---
-func Major(dev uint64) int {
- return int(((dev >> 8) & 0xfff) | ((dev >> 32) & (0xfffff000)))
-}
-
-func Minor(dev uint64) int {
- return int((dev & 0xff) | ((dev >> 12) & (0xffffff00)))
-}
-
-func GetFileStat(p string) (uid int, gid int, major int, minor int,
- inode uint64, nlink int, err error) {
+func GetFileStat(p string) (uid int, gid int, major uint32, minor uint32, inode uint64, nlink int, err error) {
var stat unix.Stat_t
err = unix.Lstat(p, &stat)
if err != nil {
@@ -38,11 +29,9 @@ func GetFileStat(p string) (uid int, gid int, major int, minor int,
gid = int(stat.Gid)
inode = uint64(stat.Ino)
nlink = int(stat.Nlink)
- major = -1
- minor = -1
if stat.Mode&unix.S_IFBLK != 0 || stat.Mode&unix.S_IFCHR != 0 {
- major = Major(stat.Rdev)
- minor = Minor(stat.Rdev)
+ major = unix.Major(stat.Rdev)
+ minor = unix.Minor(stat.Rdev)
}
return
From d15bc94b325f22165e65300f00b04174fa44dc18 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 10:49:55 +0100
Subject: [PATCH 14/26] storage/quota/projectquota: Moves use of Major and
Minor functions to unix package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/storage/quota/projectquota.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lxd/storage/quota/projectquota.go b/lxd/storage/quota/projectquota.go
index e6efdc3237..f96032009d 100644
--- a/lxd/storage/quota/projectquota.go
+++ b/lxd/storage/quota/projectquota.go
@@ -164,8 +164,8 @@ func devForPath(path string) (string, error) {
return "", err
}
- devMajor := shared.Major(stat.Dev)
- devMinor := shared.Minor(stat.Dev)
+ devMajor := unix.Major(stat.Dev)
+ devMinor := unix.Minor(stat.Dev)
// Parse mountinfo for it
mountinfo, err := os.Open("/proc/self/mountinfo")
From a8432d86e417659208fdba3655df3dd618131d74 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 10:50:53 +0100
Subject: [PATCH 15/26] shared/containerwriter/container/tar/writer: Updates
use of GetFileStat
GetFileStat now uses the unix package to get file device major and minor number and returns them as uint32.
As such we can no longer use -1 to indicate whether or not the device has a major or minor number.
However in WriteFile the default value for the DevMajor and DevMinor value in the Header struct seem to be 0 which will be the same value as returned by GetFileStat when the file is not a char or block device.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
shared/containerwriter/container_tar_writer.go | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/shared/containerwriter/container_tar_writer.go b/shared/containerwriter/container_tar_writer.go
index 56f5937f2a..e0a911c53b 100644
--- a/shared/containerwriter/container_tar_writer.go
+++ b/shared/containerwriter/container_tar_writer.go
@@ -27,7 +27,8 @@ func NewContainerTarWriter(writer io.Writer, idmapSet *idmap.IdmapSet) *Containe
func (ctw *ContainerTarWriter) WriteFile(offset int, path string, fi os.FileInfo) error {
var err error
- var major, minor, nlink int
+ var major, minor uint32
+ var nlink int
var ino uint64
link := ""
@@ -72,10 +73,8 @@ func (ctw *ContainerTarWriter) WriteFile(offset int, path string, fi os.FileInfo
}
}
- if major != -1 {
- hdr.Devmajor = int64(major)
- hdr.Devminor = int64(minor)
- }
+ hdr.Devmajor = int64(major)
+ hdr.Devminor = int64(minor)
// If it's a hardlink we've already seen use the old name
if fi.Mode().IsRegular() && nlink > 1 {
From 99fef9b51d76196f02ae297ea8a3958b5ce5cf05 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 9 Aug 2019 16:26:04 +0100
Subject: [PATCH 16/26] device/usb: Adds USB device implementation
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/usb.go | 256 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 256 insertions(+)
create mode 100644 lxd/device/usb.go
diff --git a/lxd/device/usb.go b/lxd/device/usb.go
new file mode 100644
index 0000000000..7ab85d18a9
--- /dev/null
+++ b/lxd/device/usb.go
@@ -0,0 +1,256 @@
+package device
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path"
+ "strings"
+
+ "github.com/lxc/lxd/lxd/device/config"
+ "github.com/lxc/lxd/lxd/instance"
+ "github.com/lxc/lxd/shared"
+)
+
+type usb struct {
+ deviceCommon
+}
+
+// validateConfig checks the supplied config for correctness.
+func (d *usb) validateConfig() error {
+ if d.instance.Type() != instance.TypeContainer {
+ return ErrUnsupportedDevType
+ }
+
+ rules := map[string]func(string) error{
+ "vendorid": shared.IsAny,
+ "productid": shared.IsAny,
+ "id": shared.IsAny,
+ "uid": shared.IsUnixUserID,
+ "gid": shared.IsUnixUserID,
+ "mode": shared.IsOctalFileMode,
+ }
+
+ err := config.ValidateDevice(rules, d.config)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// validateEnvironment checks the runtime environment for correctness.
+func (d *usb) validateEnvironment() error {
+ return nil
+}
+
+// Register is run when the device is added to the instance and when LXD starts.
+func (d *usb) Register() (*RegisterConfig, error) {
+ regConf := RegisterConfig{
+ USBHooks: []func(USBDevice) (*RunConfig, error){d.eventHandler},
+ }
+
+ return ®Conf, nil
+}
+
+// Start is run when the device is added to the instance.
+func (d *usb) Start() (*RunConfig, error) {
+ err := d.validateEnvironment()
+ if err != nil {
+ return nil, err
+ }
+
+ /*
+ if usbs == nil {
+ usbs, err = deviceLoadUsb()
+ if err != nil {
+ return err
+ }
+ }
+
+ for _, usb := range usbs {
+ if (m["vendorid"] != "" && usb.vendor != m["vendorid"]) || (m["productid"] != "" && usb.product != m["productid"]) {
+ continue
+ }
+
+ err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, usb.major, usb.minor, usb.path, false)
+ if err != nil {
+ logger.Error("Failed to insert usb device", log.Ctx{"err": err, "usb": usb, "instance": c.Name()})
+ }
+ }
+ */
+
+ /*
+ m := devices[name]
+ if m["type"] != "usb" {
+ continue
+ }
+
+ if (m["vendorid"] != "" && m["vendorid"] != usb.Vendor) || (m["productid"] != "" && m["productid"] != usb.Product) {
+ continue
+ }
+
+ if usb.Action == "add" {
+ err := c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", name), m, usb.Major, usb.Minor, usb.Path, false)
+ if err != nil {
+ logger.Error("Failed to create usb device", log.Ctx{"err": err, "usb": usb, "instance": c.Name()})
+ return
+ }
+ } else if usb.Action == "remove" {
+ err := c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", name), m, usb.Major, usb.Minor, usb.Path)
+ if err != nil {
+ logger.Error("Failed to remove usb device", log.Ctx{"err": err, "usb": usb, "instance": c.Name()})
+ return
+ }
+ }
+
+
+ */
+
+ runConf := RunConfig{}
+ return &runConf, nil
+}
+
+// Stop is run when the device is removed from the instance.
+func (d *usb) Stop() (*RunConfig, error) {
+ runConf := RunConfig{
+ PostHooks: []func() error{d.postStop},
+ }
+
+ /*
+ if usbs == nil {
+ usbs, err = deviceLoadUsb()
+ if err != nil {
+ return err
+ }
+ }
+
+ /* if the device isn't present, we don't need to remove it
+ for _, usb := range usbs {
+ if (m["vendorid"] != "" && usb.vendor != m["vendorid"]) || (m["productid"] != "" && usb.product != m["productid"]) {
+ continue
+ }
+
+ err := c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, usb.major, usb.minor, usb.path)
+ if err != nil {
+ return err
+ }
+ }
+ */
+
+ return &runConf, nil
+}
+
+// eventHandler when a USB event occurs.
+func (d *usb) eventHandler(event USBDevice) (*RunConfig, error) {
+ runConf := RunConfig{}
+
+ // Check if event matches criteria for this device, if not return.
+ if (d.config["vendorid"] != "" && d.config["vendorid"] != event.Vendor) || (d.config["productid"] != "" && d.config["productid"] != event.Product) {
+ return nil, nil
+ }
+
+ if event.Action == "add" {
+ err := unixDeviceSetupCharNum(d.state, d.instance.DevicesPath(), "unix", d.name, d.config, event.Major, event.Minor, event.Path, false, &runConf)
+ if err != nil {
+ return nil, err
+ }
+ } else if event.Action == "remove" {
+ err := unixDeviceRemove(d.instance.DevicesPath(), "unix", d.name, &runConf)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ /*
+ ueventArray := make([]string, 4)
+ ueventArray[0] = "forkuevent"
+ ueventArray[1] = "inject"
+ ueventArray[2] = fmt.Sprintf("%d", c.InitPID())
+ ueventArray[3] = fmt.Sprintf("%d", usb.UeventLen)
+ ueventArray = append(ueventArray, usb.UeventParts...)
+ shared.RunCommand(s.OS.ExecPath, ueventArray...)
+ */
+
+ return &runConf, nil
+}
+
+// postStop is run after the device is removed from the instance.
+func (d *usb) postStop() error {
+ return nil
+}
+
+// loadUsb scans the host machine for USB devices.
+func (d *usb) loadUsb() ([]USBDevice, error) {
+ result := []USBDevice{}
+
+ ents, err := ioutil.ReadDir(devPath)
+ if err != nil {
+ /* if there are no USB devices, let's render an empty list,
+ * i.e. no usb devices */
+ if os.IsNotExist(err) {
+ return result, nil
+ }
+ return nil, err
+ }
+
+ for _, ent := range ents {
+ values, err := d.loadRawValues(path.Join(devPath, ent.Name()))
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+
+ return []USBDevice{}, err
+ }
+
+ parts := strings.Split(values["dev"], ":")
+ if len(parts) != 2 {
+ return []USBDevice{}, fmt.Errorf("invalid device value %s", values["dev"])
+ }
+
+ usb, err := USBDeviceLoad(
+ "add",
+ values["idVendor"],
+ values["idProduct"],
+ parts[0],
+ parts[1],
+ values["busnum"],
+ values["devnum"],
+ values["devname"],
+ []string{},
+ 0,
+ )
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+ return nil, err
+ }
+
+ result = append(result, usb)
+ }
+
+ return result, nil
+}
+
+func (d *usb) loadRawValues(p string) (map[string]string, error) {
+ values := map[string]string{
+ "idVendor": "",
+ "idProduct": "",
+ "dev": "",
+ "busnum": "",
+ "devnum": "",
+ }
+
+ for k := range values {
+ v, err := ioutil.ReadFile(path.Join(p, k))
+ if err != nil {
+ return nil, err
+ }
+
+ values[k] = strings.TrimSpace(string(v))
+ }
+
+ return values, nil
+}
From ec33f48cc7777bbd4435effe5247b5cf3afe52e4 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 9 Aug 2019 16:26:20 +0100
Subject: [PATCH 17/26] device/device/utils/usb: Adds USB util functions
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_utils_usb.go | 66 ++++++++++++++++++++++++++++++++++
1 file changed, 66 insertions(+)
create mode 100644 lxd/device/device_utils_usb.go
diff --git a/lxd/device/device_utils_usb.go b/lxd/device/device_utils_usb.go
new file mode 100644
index 0000000000..b1fe790773
--- /dev/null
+++ b/lxd/device/device_utils_usb.go
@@ -0,0 +1,66 @@
+package device
+
+import (
+ "fmt"
+ "path/filepath"
+ "strconv"
+)
+
+// devPath is the path where USB devices can be enumerated.
+const devPath = "/sys/bus/usb/devices"
+
+// USBDevice represents the properties of a USB device.
+type USBDevice struct {
+ Action string
+
+ Vendor string
+ Product string
+
+ Path string
+ Major uint32
+ Minor uint32
+ UeventParts []string
+ UeventLen int
+}
+
+// USBDeviceLoad instantiates a new USBDevice struct.
+func USBDeviceLoad(action string, vendor string, product string, major string, minor string, busnum string, devnum string, devname string, ueventParts []string, ueventLen int) (USBDevice, error) {
+ majorInt, err := strconv.ParseUint(major, 10, 32)
+ if err != nil {
+ return USBDevice{}, err
+ }
+
+ minorInt, err := strconv.ParseUint(minor, 10, 32)
+ if err != nil {
+ return USBDevice{}, err
+ }
+
+ path := devname
+ if devname == "" {
+ busnumInt, err := strconv.Atoi(busnum)
+ if err != nil {
+ return USBDevice{}, err
+ }
+
+ devnumInt, err := strconv.Atoi(devnum)
+ if err != nil {
+ return USBDevice{}, err
+ }
+ path = fmt.Sprintf("/dev/bus/usb/%03d/%03d", busnumInt, devnumInt)
+ } else {
+ if !filepath.IsAbs(devname) {
+ path = fmt.Sprintf("/dev/%s", devname)
+ }
+ }
+
+ return USBDevice{
+ action,
+ vendor,
+ product,
+ path,
+ uint32(majorInt),
+ uint32(minorInt),
+ ueventParts,
+ ueventLen,
+ }, nil
+}
From 02c386fb6a028a814efcce324226a59ac7098515 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 9 Aug 2019 16:26:48 +0100
Subject: [PATCH 18/26] container/lxc: Updates to use USB from device package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container_lxc.go | 58 --------------------------------------------
1 file changed, 58 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 6a97f78242..c2d3f05c7c 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -2548,7 +2548,6 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
c.removeUnixDevices()
c.removeDiskDevices()
- var usbs []usbDevice
diskDevices := map[string]config.Device{}
// Create the devices
@@ -2597,24 +2596,6 @@ func (c *containerLXC) startCommon() (string, []func() error, error) {
}
}
}
- } else if m["type"] == "usb" {
- if usbs == nil {
- usbs, err = deviceLoadUsb()
- if err != nil {
- return "", postStartHooks, err
- }
- }
-
- for _, usb := range usbs {
- if (m["vendorid"] != "" && usb.vendor != m["vendorid"]) || (m["productid"] != "" && usb.product != m["productid"]) {
- continue
- }
-
- err := c.setupUnixDevice(fmt.Sprintf("unix.%s", k), m, usb.major, usb.minor, usb.path, shared.IsTrue(m["required"]), false)
- if err != nil {
- return "", postStartHooks, err
- }
- }
} else if m["type"] == "disk" {
if m["path"] != "/" {
diskDevices[k] = m
@@ -4938,8 +4919,6 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
}
}
- var usbs []usbDevice
-
// Live update the devices
for k, m := range removeDevices {
if shared.StringInSlice(m["type"], []string{"unix-char", "unix-block"}) {
@@ -4962,25 +4941,6 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
if err != nil {
return err
}
- } else if m["type"] == "usb" {
- if usbs == nil {
- usbs, err = deviceLoadUsb()
- if err != nil {
- return err
- }
- }
-
- /* if the device isn't present, we don't need to remove it */
- for _, usb := range usbs {
- if (m["vendorid"] != "" && usb.vendor != m["vendorid"]) || (m["productid"] != "" && usb.product != m["productid"]) {
- continue
- }
-
- err := c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, usb.major, usb.minor, usb.path)
- if err != nil {
- return err
- }
- }
}
}
@@ -4995,24 +4955,6 @@ func (c *containerLXC) Update(args db.ContainerArgs, userRequested bool) error {
}
} else if m["type"] == "disk" && m["path"] != "/" {
diskDevices[k] = m
- } else if m["type"] == "usb" {
- if usbs == nil {
- usbs, err = deviceLoadUsb()
- if err != nil {
- return err
- }
- }
-
- for _, usb := range usbs {
- if (m["vendorid"] != "" && usb.vendor != m["vendorid"]) || (m["productid"] != "" && usb.product != m["productid"]) {
- continue
- }
-
- err = c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", k), m, usb.major, usb.minor, usb.path, false)
- if err != nil {
- logger.Error("Failed to insert usb device", log.Ctx{"err": err, "usb": usb, "container": c.Name()})
- }
- }
}
}
From c458e12c97489f9137f697a3cb10901f8017b1d8 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 9 Aug 2019 16:27:07 +0100
Subject: [PATCH 19/26] device: Links usb devices to use USB implementation
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/lxd/device/device.go b/lxd/device/device.go
index 5c6833e1e6..d2cfcc1ead 100644
--- a/lxd/device/device.go
+++ b/lxd/device/device.go
@@ -13,6 +13,7 @@ var devTypes = map[string]func(config.Device) device{
"infiniband": infinibandLoadByType,
"proxy": func(c config.Device) device { return &proxy{} },
"gpu": func(c config.Device) device { return &gpu{} },
+ "usb": func(c config.Device) device { return &usb{} },
}
// VolatileSetter is a function that accepts one or more key/value strings to save into the LXD
From 1b5372c8b6ec8828e24d0affd21ef801bc7a358e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Fri, 9 Aug 2019 16:27:30 +0100
Subject: [PATCH 20/26] devices: Moves USB logic to device package
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/devices.go | 153 ++++---------------------------------------------
1 file changed, 12 insertions(+), 141 deletions(-)
diff --git a/lxd/devices.go b/lxd/devices.go
index 2e962d4389..2edb989b4e 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -48,61 +48,8 @@ func (c deviceTaskCPUs) Len() int { return len(c) }
func (c deviceTaskCPUs) Less(i, j int) bool { return *c[i].count < *c[j].count }
func (c deviceTaskCPUs) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
-type usbDevice struct {
- action string
- vendor string
- product string
-
- path string
- major int
- minor int
- ueventParts []string
- ueventLen int
-}
-
-func createUSBDevice(action string, vendor string, product string, major string, minor string, busnum string, devnum string, devname string, ueventParts []string, ueventLen int) (usbDevice, error) {
- majorInt, err := strconv.Atoi(major)
- if err != nil {
- return usbDevice{}, err
- }
-
- minorInt, err := strconv.Atoi(minor)
- if err != nil {
- return usbDevice{}, err
- }
-
- path := devname
- if devname == "" {
- busnumInt, err := strconv.Atoi(busnum)
- if err != nil {
- return usbDevice{}, err
- }
-
- devnumInt, err := strconv.Atoi(devnum)
- if err != nil {
- return usbDevice{}, err
- }
- path = fmt.Sprintf("/dev/bus/usb/%03d/%03d", busnumInt, devnumInt)
- } else {
- if !filepath.IsAbs(devname) {
- path = fmt.Sprintf("/dev/%s", devname)
- }
- }
-
- return usbDevice{
- action,
- vendor,
- product,
- path,
- majorInt,
- minorInt,
- ueventParts,
- ueventLen,
- }, nil
-}
-
-func deviceNetlinkListener() (chan []string, chan []string, chan usbDevice, error) {
+func deviceNetlinkListener() (chan []string, chan []string, chan device.USBDevice, error) {
NETLINK_KOBJECT_UEVENT := 15
UEVENT_BUFFER_SIZE := 2048
@@ -127,9 +74,9 @@ func deviceNetlinkListener() (chan []string, chan []string, chan usbDevice, erro
chCPU := make(chan []string, 1)
chNetwork := make(chan []string, 0)
- chUSB := make(chan usbDevice)
+ chUSB := make(chan device.USBDevice)
- go func(chCPU chan []string, chNetwork chan []string, chUSB chan usbDevice) {
+ go func(chCPU chan []string, chNetwork chan []string, chUSB chan device.USBDevice) {
b := make([]byte, UEVENT_BUFFER_SIZE*2)
for {
r, err := unix.Read(fd, b)
@@ -224,7 +171,7 @@ func deviceNetlinkListener() (chan []string, chan []string, chan usbDevice, erro
return strings.Repeat("0", l-len(s)) + s
}
- usb, err := createUSBDevice(
+ usb, err := device.USBDeviceLoad(
props["ACTION"],
/* udev doesn't zero pad these, while
* everything else does, so let's zero pad them
@@ -507,7 +454,7 @@ func deviceNetworkPriority(s *state.State, netif string) {
return
}
-func deviceUSBEvent(s *state.State, usb usbDevice) {
+func deviceUSBEvent(s *state.State, usb device.USBDevice) {
containers, err := containerLoadNodeAll(s)
if err != nil {
logger.Error("Problem loading containers list", log.Ctx{"err": err})
@@ -532,18 +479,18 @@ func deviceUSBEvent(s *state.State, usb usbDevice) {
continue
}
- if (m["vendorid"] != "" && m["vendorid"] != usb.vendor) || (m["productid"] != "" && m["productid"] != usb.product) {
+ if (m["vendorid"] != "" && m["vendorid"] != usb.Vendor) || (m["productid"] != "" && m["productid"] != usb.Product) {
continue
}
- if usb.action == "add" {
- err := c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", name), m, usb.major, usb.minor, usb.path, false)
+ if usb.Action == "add" {
+ err := c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", name), m, usb.Major, usb.Minor, usb.Path, false)
if err != nil {
logger.Error("Failed to create usb device", log.Ctx{"err": err, "usb": usb, "container": c.Name()})
return
}
- } else if usb.action == "remove" {
- err := c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", name), m, usb.major, usb.minor, usb.path)
+ } else if usb.Action == "remove" {
+ err := c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", name), m, usb.Major, usb.Minor, usb.Path)
if err != nil {
logger.Error("Failed to remove usb device", log.Ctx{"err": err, "usb": usb, "container": c.Name()})
return
@@ -554,8 +501,8 @@ func deviceUSBEvent(s *state.State, usb usbDevice) {
ueventArray[0] = "forkuevent"
ueventArray[1] = "inject"
ueventArray[2] = fmt.Sprintf("%d", c.InitPID())
- ueventArray[3] = fmt.Sprintf("%d", usb.ueventLen)
- ueventArray = append(ueventArray, usb.ueventParts...)
+ ueventArray[3] = fmt.Sprintf("%d", usb.UeventLen)
+ ueventArray = append(ueventArray, usb.UeventParts...)
shared.RunCommand(s.OS.ExecPath, ueventArray...)
}
}
@@ -887,82 +834,6 @@ func deviceParseDiskLimit(readSpeed string, writeSpeed string) (int64, int64, in
return readBps, readIops, writeBps, writeIops, nil
}
-const USB_PATH = "/sys/bus/usb/devices"
-
-func loadRawValues(p string) (map[string]string, error) {
- values := map[string]string{
- "idVendor": "",
- "idProduct": "",
- "dev": "",
- "busnum": "",
- "devnum": "",
- }
-
- for k := range values {
- v, err := ioutil.ReadFile(path.Join(p, k))
- if err != nil {
- return nil, err
- }
-
- values[k] = strings.TrimSpace(string(v))
- }
-
- return values, nil
-}
-
-func deviceLoadUsb() ([]usbDevice, error) {
- result := []usbDevice{}
-
- ents, err := ioutil.ReadDir(USB_PATH)
- if err != nil {
- /* if there are no USB devices, let's render an empty list,
- * i.e. no usb devices */
- if os.IsNotExist(err) {
- return result, nil
- }
- return nil, err
- }
-
- for _, ent := range ents {
- values, err := loadRawValues(path.Join(USB_PATH, ent.Name()))
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
-
- return []usbDevice{}, err
- }
-
- parts := strings.Split(values["dev"], ":")
- if len(parts) != 2 {
- return []usbDevice{}, fmt.Errorf("invalid device value %s", values["dev"])
- }
-
- usb, err := createUSBDevice(
- "add",
- values["idVendor"],
- values["idProduct"],
- parts[0],
- parts[1],
- values["busnum"],
- values["devnum"],
- values["devname"],
- []string{},
- 0,
- )
- if err != nil {
- if os.IsNotExist(err) {
- continue
- }
- return nil, err
- }
-
- result = append(result, usb)
- }
-
- return result, nil
-}
-
func deviceInotifyInit(s *state.State) (int, error) {
s.OS.InotifyWatch.Lock()
defer s.OS.InotifyWatch.Unlock()
From bf83c37be02ad644f042df874dc1cd88fc4eaf1f Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 15:31:44 +0100
Subject: [PATCH 21/26] uevents: Adds uevent usb event handling registration
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/uevents/uevents.go | 51 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 lxd/uevents/uevents.go
diff --git a/lxd/uevents/uevents.go b/lxd/uevents/uevents.go
new file mode 100644
index 0000000000..0bde0022f4
--- /dev/null
+++ b/lxd/uevents/uevents.go
@@ -0,0 +1,51 @@
+package uevents
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+
+ "github.com/lxc/lxd/lxd/device"
+ log "github.com/lxc/lxd/shared/log15"
+ "github.com/lxc/lxd/shared/logger"
+)
+
+// usbHandlers stores the event handler callbacks for USB events.
+var usbHandlers = map[string]func(device.USBDevice) error{}
+
+// usbMutex controls access to the usbHandlers map.
+var usbMutex sync.Mutex
+
+// RegisterUSBHandler registers a handler function to be called whenever a USB device event occurs.
+func RegisterUSBHandler(instance device.InstanceIdentifier, deviceName string, handler func(device.USBDevice) error) {
+ usbMutex.Lock()
+ defer usbMutex.Unlock()
+
+ // Null delimited string of project name, instance name and device name.
+ key := fmt.Sprintf("%s\000%s\000%s", instance.Project(), instance.Name(), deviceName)
+ usbHandlers[key] = handler
+}
+
+// UnregisterUSBHandler removes a registered USB handler function for a device.
+func UnregisterUSBHandler(instance device.InstanceIdentifier, deviceName string) {
+ usbMutex.Lock()
+ defer usbMutex.Unlock()
+
+ // Null delimited string of project name, instance name and device name.
+ key := fmt.Sprintf("%s\000%s\000%s", instance.Project(), instance.Name(), deviceName)
+ delete(usbHandlers, key)
+}
+
+// RunUSBHandlers executes any handlers registered for USB events.
+func RunUSBHandlers(event device.USBDevice) {
+ usbMutex.Lock()
+ defer usbMutex.Unlock()
+
+ for k, v := range usbHandlers {
+ err := v(event)
+ if err != nil {
+ KeyParts := strings.SplitN(k, "\000", 3)
+ logger.Error("Failed to handle USB event", log.Ctx{"err": err, "project": KeyParts[0], "instance": KeyParts[1], "device": KeyParts[2]})
+ }
+ }
+}
From ce82299f72c748613f1822e5f47d5bff3a574b69 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 15:44:43 +0100
Subject: [PATCH 22/26] device/device/runconfig: Typo in comment
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_runconfig.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lxd/device/device_runconfig.go b/lxd/device/device_runconfig.go
index b06bf5fe0a..d7b30073ce 100644
--- a/lxd/device/device_runconfig.go
+++ b/lxd/device/device_runconfig.go
@@ -14,7 +14,7 @@ type MountEntryItem struct {
Opts []string // Describes the mount options associated with the filesystem.
Freq int // Used by dump(8) to determine which filesystems need to be dumped. Defaults to zero (don't dump) if not present.
PassNo int // Used by fsck(8) to determine the order in which filesystem checks are done at boot time. Defaults to zero (don't fsck) if not present.
- Shift bool // Whether or not to use shifts with this mount.
+ Shift bool // Whether or not to use shiftfs with this mount.
}
// RunConfig represents LXD defined run-time config used for device setup/cleanup.
From e9992d8ecf3ffebd95bcc23da68f7abab1efceef Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 17:40:00 +0100
Subject: [PATCH 23/26] devices: Moves USB event handling to uevents
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/devices.go | 58 ++------------------------------------------------
1 file changed, 2 insertions(+), 56 deletions(-)
diff --git a/lxd/devices.go b/lxd/devices.go
index 2edb989b4e..b27ef8c45e 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -20,6 +20,7 @@ import (
"github.com/lxc/lxd/lxd/device"
"github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/lxd/sys"
+ "github.com/lxc/lxd/lxd/uevents"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/logger"
@@ -48,7 +49,6 @@ func (c deviceTaskCPUs) Len() int { return len(c) }
func (c deviceTaskCPUs) Less(i, j int) bool { return *c[i].count < *c[j].count }
func (c deviceTaskCPUs) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
-
func deviceNetlinkListener() (chan []string, chan []string, chan device.USBDevice, error) {
NETLINK_KOBJECT_UEVENT := 15
UEVENT_BUFFER_SIZE := 2048
@@ -454,60 +454,6 @@ func deviceNetworkPriority(s *state.State, netif string) {
return
}
-func deviceUSBEvent(s *state.State, usb device.USBDevice) {
- containers, err := containerLoadNodeAll(s)
- if err != nil {
- logger.Error("Problem loading containers list", log.Ctx{"err": err})
- return
- }
-
- for _, containerIf := range containers {
- c, ok := containerIf.(*containerLXC)
- if !ok {
- logger.Errorf("Got device event on non-LXC container?")
- return
- }
-
- if !c.IsRunning() {
- continue
- }
-
- devices := c.ExpandedDevices()
- for _, name := range devices.DeviceNames() {
- m := devices[name]
- if m["type"] != "usb" {
- continue
- }
-
- if (m["vendorid"] != "" && m["vendorid"] != usb.Vendor) || (m["productid"] != "" && m["productid"] != usb.Product) {
- continue
- }
-
- if usb.Action == "add" {
- err := c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", name), m, usb.Major, usb.Minor, usb.Path, false)
- if err != nil {
- logger.Error("Failed to create usb device", log.Ctx{"err": err, "usb": usb, "container": c.Name()})
- return
- }
- } else if usb.Action == "remove" {
- err := c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", name), m, usb.Major, usb.Minor, usb.Path)
- if err != nil {
- logger.Error("Failed to remove usb device", log.Ctx{"err": err, "usb": usb, "container": c.Name()})
- return
- }
- }
-
- ueventArray := make([]string, 4)
- ueventArray[0] = "forkuevent"
- ueventArray[1] = "inject"
- ueventArray[2] = fmt.Sprintf("%d", c.InitPID())
- ueventArray[3] = fmt.Sprintf("%d", usb.UeventLen)
- ueventArray = append(ueventArray, usb.UeventParts...)
- shared.RunCommand(s.OS.ExecPath, ueventArray...)
- }
- }
-}
-
func deviceEventListener(s *state.State) {
chNetlinkCPU, chNetlinkNetwork, chUSB, err := deviceNetlinkListener()
if err != nil {
@@ -543,7 +489,7 @@ func deviceEventListener(s *state.State) {
deviceNetworkPriority(s, e[0])
networkAutoAttach(s.Cluster, e[0])
case e := <-chUSB:
- deviceUSBEvent(s, e)
+ uevents.RunUSBHandlers(e)
case e := <-deviceSchedRebalance:
if len(e) != 3 {
logger.Errorf("Scheduler: received an invalid rebalance event")
From 3844caa8a07453fe7b53aa9a70b58faab70de93e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 17:41:16 +0100
Subject: [PATCH 24/26] device: Adds Register() function to device interface
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device.go | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/lxd/device/device.go b/lxd/device/device.go
index d2cfcc1ead..37c59b0a8e 100644
--- a/lxd/device/device.go
+++ b/lxd/device/device.go
@@ -36,6 +36,10 @@ type Device interface {
// It is called irrespective of whether the instance is running or not.
Add() error
+ // Register returns registration config for subscribing to events that LXD can generate.
+ // It is called after a device is added to an instance (after Add()) and when LXD starts.
+ Register() (*RegisterConfig, error)
+
// Start peforms any host-side configuration required to start the device for the instance.
// This can be when a device is plugged into a running instance or the instance is starting.
// Returns run-time configuration needed for configuring the instance with the new device.
@@ -99,6 +103,11 @@ func (d *deviceCommon) Add() error {
return nil
}
+// Register returns nil error as majority of devices don't need to do any event registration.
+func (d *deviceCommon) Register() (*RegisterConfig, error) {
+ return nil, nil
+}
+
// CanHotPlug returns true as majority of devices can be started/stopped when instance is running.
// Also returns an empty list of update fields as most devices do not support live updates.
func (d *deviceCommon) CanHotPlug() (bool, []string) {
From 1e2b7c65deb1b43574bc501991f0d83349b1a1d0 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 17:43:14 +0100
Subject: [PATCH 25/26] device/device/runconfig: Adds RegisterConfig for USB
events
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/device/device_runconfig.go | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/lxd/device/device_runconfig.go b/lxd/device/device_runconfig.go
index d7b30073ce..603e4f4612 100644
--- a/lxd/device/device_runconfig.go
+++ b/lxd/device/device_runconfig.go
@@ -24,3 +24,8 @@ type RunConfig struct {
Mounts []MountEntryItem // Mounts to setup/remove.
PostHooks []func() error // Functions to be run after device attach/detach.
}
+
+// RegisterConfig represents registration config for subscribing to events that LXD can generate.
+type RegisterConfig struct {
+ USBHooks []func(USBDevice) (*RunConfig, error) // Functions to be run after a USB event has occurred.
+}
From 3fb44cd74a40a65101ca21271f06aec2cfeda86b Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 14 Aug 2019 17:45:00 +0100
Subject: [PATCH 26/26] container/lxc: Adds support for uevents
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/container_lxc.go | 47 ++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 45 insertions(+), 2 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index c2d3f05c7c..788fe7ec85 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -36,6 +36,7 @@ import (
"github.com/lxc/lxd/lxd/project"
"github.com/lxc/lxd/lxd/state"
"github.com/lxc/lxd/lxd/template"
+ "github.com/lxc/lxd/lxd/uevents"
"github.com/lxc/lxd/lxd/util"
"github.com/lxc/lxd/shared"
"github.com/lxc/lxd/shared/api"
@@ -1887,14 +1888,53 @@ func (c *containerLXC) deviceLoad(deviceName string, rawConfig map[string]string
return d, configCopy, nil
}
-// deviceAdd loads a new device and calls its Setup() function.
+// deviceAdd loads a new device and calls its Add() function and then its Register() function.
+// Register config returned from Register() will be used to register this device to receive events.
func (c *containerLXC) deviceAdd(deviceName string, rawConfig map[string]string) error {
d, _, err := c.deviceLoad(deviceName, rawConfig)
if err != nil {
return err
}
- return d.Add()
+ err = d.Add()
+ if err != nil {
+ return err
+ }
+
+ regConf, err := d.Register()
+ if err != nil {
+ return err
+ }
+
+ // Register any USB event hooks requested.
+ if len(regConf.USBHooks) > 0 {
+ uevents.RegisterUSBHandler(c, deviceName, func(event device.USBDevice) error {
+ // Use the container context aware USB event handler to run hooks.
+ return c.deviceHandleUSBEvent(deviceName, event, regConf.USBHooks)
+ })
+ }
+
+ return nil
+}
+
+// deviceHandleUSBEvent handles USB events by running the slice of hook functions passed to it and
+// then handling any returned requests for adding or remove devices from the container.
+// The hooks are run sequentially and if one returns a non-nil error the execution is stopped and
+// the error is returned.
+func (c *containerLXC) deviceHandleUSBEvent(deviceName string, event device.USBDevice, hooks []func(device.USBDevice) (*device.RunConfig, error)) error {
+ if !c.IsRunning() {
+ return nil
+ }
+
+ for _, hook := range hooks {
+ runConf, err := hook(event)
+ if err != nil {
+ return err
+ }
+ logger.Errorf("tomp usb event result: %+v", runConf)
+ }
+
+ return nil
}
// deviceStart loads a new device and calls its Start() function.
@@ -2082,6 +2122,9 @@ func (c *containerLXC) deviceStop(deviceName string, rawConfig map[string]string
return fmt.Errorf("Device cannot be stopped when container is running")
}
+ // Unregister any USB event handlers for this device.
+ uevents.UnregisterUSBHandler(c, deviceName)
+
runConfig, err := d.Stop()
if err != nil {
return err
More information about the lxc-devel
mailing list