[lxc-devel] [lxd/master] lxd: support uevent injection for USB devices
brauner on Github
lxc-bot at linuxcontainers.org
Wed Oct 3 21:29:39 UTC 2018
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 381 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20181003/3fe79b34/attachment.bin>
-------------- next part --------------
From d1990de61e82f6b02d7a3324b0a08221f39177e0 Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at ubuntu.com>
Date: Wed, 3 Oct 2018 23:28:26 +0200
Subject: [PATCH] lxd: support uevent injection for USB devices
Closes #4874.
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
lxd/daemon.go | 8 ++
lxd/devices.go | 58 +++++++----
lxd/main.go | 4 +
lxd/main_checkfeature.go | 23 ++++-
lxd/main_forkuevent.go | 214 +++++++++++++++++++++++++++++++++++++++
lxd/main_nsexec.go | 3 +
lxd/sys/os.go | 1 +
7 files changed, 292 insertions(+), 19 deletions(-)
create mode 100644 lxd/main_forkuevent.go
diff --git a/lxd/daemon.go b/lxd/daemon.go
index 2a94c65cf9..2219ee0054 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -439,6 +439,14 @@ func (d *Daemon) init() error {
logger.Debugf("Running kernel does not support netnsid-based network retrieval")
}
+ /* Check if uevent injection is supported. */
+ d.os.UeventInjection = CanUseUeventInjection()
+ if d.os.UeventInjection {
+ logger.Debugf("Running kernel supports uevent injection")
+ } else {
+ logger.Debugf("Running kernel does not support uevent injection")
+ }
+
/* Initialize the database */
dump, err := initializeDbObject(d)
if err != nil {
diff --git a/lxd/devices.go b/lxd/devices.go
index 82540e3303..4296934d47 100644
--- a/lxd/devices.go
+++ b/lxd/devices.go
@@ -53,9 +53,11 @@ type usbDevice struct {
vendor string
product string
- path string
- major int
- minor int
+ path string
+ major int
+ minor int
+ ueventParts []string
+ ueventLen int
}
// /dev/nvidia[0-9]+
@@ -378,7 +380,7 @@ func deviceLoadGpu(all bool) ([]gpuDevice, []nvidiaGpuDevices, error) {
return gpus, nvidiaDevices, nil
}
-func createUSBDevice(action string, vendor string, product string, major string, minor string, busnum string, devnum string, devname string) (usbDevice, error) {
+func createUSBDevice(action string, vendor string, product string, major string, minor string, busnum string, devnum string, devname string, ueventParts []string, ueventLen int) (usbDevice, error) {
majorInt, err := strconv.Atoi(major)
if err != nil {
return usbDevice{}, err
@@ -414,6 +416,8 @@ func createUSBDevice(action string, vendor string, product string, major string,
path,
majorInt,
minorInt,
+ ueventParts,
+ ueventLen,
}, nil
}
@@ -448,28 +452,29 @@ func deviceNetlinkListener() (chan []string, chan []string, chan usbDevice, erro
go func(chCPU chan []string, chNetwork chan []string, chUSB chan usbDevice) {
b := make([]byte, UEVENT_BUFFER_SIZE*2)
for {
- _, err := syscall.Read(fd, b)
+ r, err := syscall.Read(fd, b)
if err != nil {
continue
}
+ ueventBuf := make([]byte, r)
+ copy(ueventBuf, b)
+ ueventLen := 0
+ ueventParts := strings.Split(string(ueventBuf), "\x00")
props := map[string]string{}
- last := 0
- for i, e := range b {
- if i == len(b) || e == 0 {
- msg := string(b[last+1 : i])
- last = i
- if len(msg) == 0 || msg == "\x00" {
- continue
- }
+ for _, part := range ueventParts {
+ if strings.HasPrefix(part, "SEQNUM=") {
+ continue
+ }
- fields := strings.SplitN(msg, "=", 2)
- if len(fields) != 2 {
- continue
- }
+ ueventLen += len(part) + 1
- props[fields[0]] = fields[1]
+ fields := strings.SplitN(part, "=", 2)
+ if len(fields) != 2 {
+ continue
}
+
+ props[fields[0]] = fields[1]
}
if props["SUBSYSTEM"] == "cpu" {
@@ -554,6 +559,8 @@ func deviceNetlinkListener() (chan []string, chan []string, chan usbDevice, erro
busnum,
devnum,
devname,
+ ueventParts[:len(ueventParts)-1],
+ ueventLen,
)
if err != nil {
logger.Error("Error reading usb device", log.Ctx{"err": err, "path": props["PHYSDEVPATH"]})
@@ -851,18 +858,31 @@ func deviceUSBEvent(s *state.State, usb usbDevice) {
continue
}
+ ueventArray := make([]string, 4)
+ ueventArray[0] = "forkuevent"
+ ueventArray[1] = "inject"
+ ueventArray[2] = fmt.Sprintf("%d", c.InitPID())
+ ueventArray[3] = fmt.Sprintf("%d", usb.ueventLen)
+ ueventArray = append(ueventArray, usb.ueventParts...)
+
if usb.action == "add" {
err := c.insertUnixDeviceNum(fmt.Sprintf("unix.%s", name), m, usb.major, usb.minor, usb.path, false)
if err != nil {
logger.Error("Failed to create usb device", log.Ctx{"err": err, "usb": usb, "container": c.Name()})
return
}
+ shared.RunCommand(s.OS.ExecPath, ueventArray...)
} else if usb.action == "remove" {
err := c.removeUnixDeviceNum(fmt.Sprintf("unix.%s", name), m, usb.major, usb.minor, usb.path)
if err != nil {
logger.Error("Failed to remove usb device", log.Ctx{"err": err, "usb": usb, "container": c.Name()})
return
}
+ shared.RunCommand(s.OS.ExecPath, ueventArray...)
+ } else if usb.action == "bind" {
+ shared.RunCommand(s.OS.ExecPath, ueventArray...)
+ } else if usb.action == "change" {
+ shared.RunCommand(s.OS.ExecPath, ueventArray...)
} else {
logger.Error("Unknown action for usb device", log.Ctx{"usb": usb})
continue
@@ -1392,6 +1412,8 @@ func deviceLoadUsb() ([]usbDevice, error) {
values["busnum"],
values["devnum"],
values["devname"],
+ []string{},
+ 0,
)
if err != nil {
if os.IsNotExist(err) {
diff --git a/lxd/main.go b/lxd/main.go
index e4ccec708c..49b0f6520b 100644
--- a/lxd/main.go
+++ b/lxd/main.go
@@ -124,6 +124,10 @@ func main() {
forkstartCmd := cmdForkstart{global: &globalCmd}
app.AddCommand(forkstartCmd.Command())
+ // forkuevent sub-command
+ forkueventCmd := cmdForkuevent{global: &globalCmd}
+ app.AddCommand(forkueventCmd.Command())
+
// import sub-command
importCmd := cmdImport{global: &globalCmd}
app.AddCommand(importCmd.Command())
diff --git a/lxd/main_checkfeature.go b/lxd/main_checkfeature.go
index 9b7b3d443b..ea4aa8e0c2 100644
--- a/lxd/main_checkfeature.go
+++ b/lxd/main_checkfeature.go
@@ -22,8 +22,11 @@ import (
#include "../shared/netns_getifaddrs.c"
bool netnsid_aware = false;
+bool uevent_aware = false;
char errbuf[4096];
+extern int inject_uevent(const char *uevent, size_t len);
+
static int netns_set_nsid(int fd)
{
int sockfd, ret;
@@ -64,7 +67,8 @@ static int netns_set_nsid(int fd)
return 0;
}
-void checkfeature() {
+void is_netnsid_aware()
+{
int netnsid, ret;
struct netns_ifaddrs *ifaddrs;
int hostnetns_fd = -1, newnetns_fd = -1;
@@ -118,6 +122,19 @@ on_error:
close(newnetns_fd);
}
+void is_uevent_aware()
+{
+ if (inject_uevent("dummy", 6) < 0)
+ _exit(1);
+
+ uevent_aware = true;
+}
+
+void checkfeature() {
+ is_netnsid_aware();
+ is_uevent_aware();
+}
+
static bool is_empty_string(char *s)
{
return (errbuf[0] == '\0');
@@ -133,3 +150,7 @@ func CanUseNetnsGetifaddrs() bool {
return bool(C.netnsid_aware)
}
+
+func CanUseUeventInjection() bool {
+ return bool(C.uevent_aware)
+}
diff --git a/lxd/main_forkuevent.go b/lxd/main_forkuevent.go
new file mode 100644
index 0000000000..0710264013
--- /dev/null
+++ b/lxd/main_forkuevent.go
@@ -0,0 +1,214 @@
+package main
+
+import (
+ "github.com/spf13/cobra"
+)
+
+/*
+
+#define _GNU_SOURCE
+#include <asm/types.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <linux/netlink.h>
+#include <linux/rtnetlink.h>
+#include <sched.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "../shared/network.c"
+
+#ifndef UEVENT_SEND
+#define UEVENT_SEND 16
+#endif
+
+extern char *advance_arg(bool required);
+extern void attach_userns(int pid);
+extern int dosetns(int pid, char *nstype);
+
+struct nlmsg {
+ struct nlmsghdr *nlmsghdr;
+ ssize_t cap;
+};
+
+static struct nlmsg *nlmsg_alloc(size_t size)
+{
+ struct nlmsg *nlmsg;
+ size_t len = NLMSG_HDRLEN + NLMSG_ALIGN(size);
+
+ nlmsg = (struct nlmsg *)malloc(sizeof(struct nlmsg));
+ if (!nlmsg)
+ return NULL;
+
+ nlmsg->nlmsghdr = (struct nlmsghdr *)malloc(len);
+ if (!nlmsg->nlmsghdr)
+ goto errout;
+
+ memset(nlmsg->nlmsghdr, 0, len);
+ nlmsg->cap = len;
+ nlmsg->nlmsghdr->nlmsg_len = NLMSG_HDRLEN;
+
+ return nlmsg;
+errout:
+ free(nlmsg);
+ return NULL;
+}
+
+static void *nlmsg_reserve_unaligned(struct nlmsg *nlmsg, size_t len)
+{
+ char *buf;
+ size_t nlmsg_len = nlmsg->nlmsghdr->nlmsg_len;
+ size_t tlen = len;
+
+ if ((ssize_t)(nlmsg_len + tlen) > nlmsg->cap)
+ return NULL;
+
+ buf = ((char *)(nlmsg->nlmsghdr)) + nlmsg_len;
+ nlmsg->nlmsghdr->nlmsg_len += tlen;
+
+ if (tlen > len)
+ memset(buf + len, 0, tlen - len);
+
+ return buf;
+}
+
+int inject_uevent(const char *uevent, size_t len)
+{
+ int ret, sock_fd;
+ char *umsg = NULL;
+ struct nlmsg *nlmsg = NULL;
+
+ sock_fd = netlink_open(NETLINK_KOBJECT_UEVENT);
+ if (sock_fd < 0) {
+ return -1;
+ }
+
+ nlmsg = nlmsg_alloc(len);
+ if (!nlmsg) {
+ ret = -1;
+ goto on_error;
+ }
+
+ nlmsg->nlmsghdr->nlmsg_flags = NLM_F_ACK | NLM_F_REQUEST;
+ nlmsg->nlmsghdr->nlmsg_type = UEVENT_SEND;
+ nlmsg->nlmsghdr->nlmsg_pid = 0;
+
+ umsg = nlmsg_reserve_unaligned(nlmsg, len);
+ if (!umsg) {
+ ret = -1;
+ goto on_error;
+ }
+
+ memcpy(umsg, uevent, len);
+
+ ret = netlink_transaction(sock_fd, nlmsg->nlmsghdr, nlmsg->nlmsghdr);
+ if (ret < 0) {
+ ret = -1;
+ goto on_error;
+ }
+
+ ret = 0;
+
+on_error:
+ close(sock_fd);
+ free(nlmsg);
+ return ret;
+}
+
+void forkuevent() {
+ char *uevent = NULL;
+ char *cur = NULL;
+ pid_t pid = 0;
+ size_t len = 0;
+
+ cur = advance_arg(false);
+ if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) {
+ fprintf(stderr, "Error: Missing PID\n");
+ _exit(1);
+ }
+
+ // Get the pid
+ cur = advance_arg(false);
+ if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) {
+ fprintf(stderr, "Error: Missing PID\n");
+ _exit(1);
+ }
+ pid = atoi(cur);
+
+ // Get the size
+ cur = advance_arg(false);
+ if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) {
+ fprintf(stderr, "Error: Missing uevent length\n");
+ _exit(1);
+ }
+ len = atoi(cur);
+
+ // Get the uevent
+ cur = advance_arg(false);
+ if (cur == NULL || (strcmp(cur, "--help") == 0 || strcmp(cur, "--version") == 0 || strcmp(cur, "-h") == 0)) {
+ fprintf(stderr, "Error: Missing uevent\n");
+ _exit(1);
+ }
+ uevent = cur;
+
+ // Check that we're root
+ if (geteuid() != 0) {
+ fprintf(stderr, "Error: forkuevent requires root privileges\n");
+ _exit(1);
+ }
+
+ attach_userns(pid);
+
+ if (dosetns(pid, "net") < 0) {
+ fprintf(stderr, "Failed to setns to container network namespace: %s\n", strerror(errno));
+ _exit(1);
+ }
+
+ if (inject_uevent(uevent, len) < 0) {
+ fprintf(stderr, "Failed to inject uevent\n");
+ _exit(1);
+ }
+}
+*/
+// #cgo CFLAGS: -std=gnu11 -Wvla
+import "C"
+
+type cmdForkuevent struct {
+ global *cmdGlobal
+}
+
+func (c *cmdForkuevent) Command() *cobra.Command {
+ // Main subcommand
+ cmd := &cobra.Command{}
+ cmd.Use = "forkuevent"
+ cmd.Short = "Inject uevents into container's network namespace"
+ cmd.Long = `Description:
+ Inject uevent into a container's network namespace
+
+ This internal command is used to inject uevents into unprivileged container's
+ network namespaces.
+`
+ cmd.Hidden = true
+
+ // pull
+ cmdInject := &cobra.Command{}
+ cmdInject.Use = "inject <PID> <len> <uevent>"
+ cmdInject.Args = cobra.ExactArgs(3)
+ cmdInject.RunE = c.Run
+ cmd.AddCommand(cmdInject)
+
+ return cmd
+}
+
+func (c *cmdForkuevent) Run(cmd *cobra.Command, args []string) error {
+ return nil
+}
diff --git a/lxd/main_nsexec.go b/lxd/main_nsexec.go
index 56fbf2fd46..c03a5d24f0 100644
--- a/lxd/main_nsexec.go
+++ b/lxd/main_nsexec.go
@@ -39,6 +39,7 @@ extern void forkfile();
extern void forkmount();
extern void forknet();
extern void forkproxy();
+extern void forkuevent();
// Command line parsing and tracking
#define CMDLINE_SIZE (8 * PATH_MAX)
@@ -246,6 +247,8 @@ __attribute__((constructor)) void init(void) {
forknet();
else if (strcmp(cmdline_cur, "forkproxy") == 0)
forkproxy();
+ else if (strcmp(cmdline_cur, "forkuevent") == 0)
+ forkuevent();
else if (strncmp(cmdline_cur, "-", 1) == 0 || strcmp(cmdline_cur, "daemon") == 0)
checkfeature();
}
diff --git a/lxd/sys/os.go b/lxd/sys/os.go
index 9e93da327a..3d28f0e7a9 100644
--- a/lxd/sys/os.go
+++ b/lxd/sys/os.go
@@ -59,6 +59,7 @@ type OS struct {
CGroupSwapAccounting bool
InotifyWatch InotifyInfo
NetnsGetifaddrs bool
+ UeventInjection bool
MockMode bool // If true some APIs will be mocked (for testing)
}
More information about the lxc-devel
mailing list