[lxc-devel] [lxd/master] VM: forklimits
tomponline on Github
lxc-bot at linuxcontainers.org
Wed Jan 29 16:08:34 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 480 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200129/29dce47b/attachment.bin>
-------------- next part --------------
From 679bad01e26cdc46c46b3f28aa3ad1e13e499f07 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 29 Jan 2020 16:03:29 +0000
Subject: [PATCH 1/3] lxd/main: Adds cmdGlobal.rawArgs function
Allows access to args that haven't been modified by cobra, allowing access to "naked" flag arguments, such as "--".
rawArgs uses os.Args but only returns arguments that occur after the supplied command name.
E.g. "lxd mycmd arg1 -- arg2" would return []string{"arg1","--","arg2"}.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/main.go | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lxd/main.go b/lxd/main.go
index 1c6299e727..bdb30f38b7 100644
--- a/lxd/main.go
+++ b/lxd/main.go
@@ -71,6 +71,17 @@ func (c *cmdGlobal) Run(cmd *cobra.Command, args []string) error {
return nil
}
+// rawArgs returns the raw unprocessed arguments from os.Args after the command name arg is found.
+func (c *cmdGlobal) rawArgs(cmd *cobra.Command) []string {
+ for i, arg := range os.Args {
+ if arg == cmd.Name() && len(os.Args)-1 > i {
+ return os.Args[i+1:]
+ }
+ }
+
+ return []string{}
+}
+
func main() {
// daemon command (main)
daemonCmd := cmdDaemon{}
From 8bf0b5ae89979f11ed3bfbdc66433a046e746b30 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 29 Jan 2020 16:05:33 +0000
Subject: [PATCH 2/3] lxd: Adds forklimits command
The forklimits command is used to spawn processes with local rlimits specified.
You can also pass through file descriptors.
This command was added to launch qemu VM processes with increased memlock limits.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/main.go | 4 ++
lxd/main_forklimits.go | 143 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 147 insertions(+)
create mode 100644 lxd/main_forklimits.go
diff --git a/lxd/main.go b/lxd/main.go
index bdb30f38b7..83a3260ea8 100644
--- a/lxd/main.go
+++ b/lxd/main.go
@@ -131,6 +131,10 @@ func main() {
forkfileCmd := cmdForkfile{global: &globalCmd}
app.AddCommand(forkfileCmd.Command())
+ // forklimits sub-command
+ forklimitsCmd := cmdForklimits{global: &globalCmd}
+ app.AddCommand(forklimitsCmd.Command())
+
// forkmigrate sub-command
forkmigrateCmd := cmdForkmigrate{global: &globalCmd}
app.AddCommand(forkmigrateCmd.Command())
diff --git a/lxd/main_forklimits.go b/lxd/main_forklimits.go
new file mode 100644
index 0000000000..6c0c7e53c1
--- /dev/null
+++ b/lxd/main_forklimits.go
@@ -0,0 +1,143 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "golang.org/x/sys/unix"
+
+ "github.com/spf13/cobra"
+)
+
+var reLimitsArg = regexp.MustCompile(`^limit=(\w+):(\w+):(\w+)$`)
+
+type cmdForklimits struct {
+ global *cmdGlobal
+}
+
+func (c *cmdForklimits) Command() *cobra.Command {
+ // Main subcommand
+ cmd := &cobra.Command{}
+ cmd.Use = "forklimits [fd=<number>...] [limit=<name>:<softlimit>:<hardlimit>...] -- <command> <args...>"
+ cmd.Short = "Execute a task inside the container"
+ cmd.Long = `Description:
+ Execute a command with specific limits set.
+
+ This internal command is used to spawn a command with limits set. It can also pass through one or more filed escriptors specified by fd=n arguments.
+ These are passed through in the order they are specified.
+`
+ cmd.RunE = c.Run
+ cmd.Hidden = true
+
+ return cmd
+}
+
+func (c *cmdForklimits) Run(cmd *cobra.Command, _ []string) error {
+ // Use raw args instead of cobra passed args, as we need to access the "--" argument.
+ args := c.global.rawArgs(cmd)
+
+ if len(args) == 0 {
+ cmd.Help()
+ return nil
+ }
+
+ // Only root should run this
+ if os.Geteuid() != 0 {
+ return fmt.Errorf("This must be run as root")
+ }
+
+ type limit struct {
+ name string
+ soft string
+ hard string
+ }
+
+ var limits []limit
+ var fds []uintptr
+ var cmdParts []string
+
+ for i, arg := range args {
+ matches := reLimitsArg.FindStringSubmatch(arg)
+ if len(matches) == 4 {
+ limits = append(limits, limit{
+ name: matches[1],
+ soft: matches[2],
+ hard: matches[3],
+ })
+ } else if strings.HasPrefix(arg, "fd=") {
+ fdParts := strings.SplitN(arg, "=", 2)
+ fdNum, err := strconv.Atoi(fdParts[1])
+ if err != nil {
+ cmd.Help()
+ return fmt.Errorf("Invalid file descriptor number")
+ }
+ fds = append(fds, uintptr(fdNum))
+ } else if arg == "--" {
+ if len(args)-1 > i {
+ cmdParts = args[i+1:]
+ }
+ break // No more passing of arguments needed.
+ } else {
+ cmd.Help()
+ return fmt.Errorf("Unrecognised argument")
+ }
+ }
+
+ // Setup rlimits.
+ for _, limit := range limits {
+ var resource int
+ var rLimit unix.Rlimit
+
+ if limit.name == "memlock" {
+ resource = unix.RLIMIT_MEMLOCK
+ } else {
+ return fmt.Errorf("Unsupported limit type: %q", limit.name)
+ }
+
+ if limit.soft == "unlimited" {
+ rLimit.Cur = unix.RLIM_INFINITY
+ } else {
+ softLimit, err := strconv.ParseUint(limit.soft, 10, 64)
+ if err != nil {
+ return fmt.Errorf("Invalid soft limit for %q", limit.name)
+ }
+ rLimit.Cur = softLimit
+ }
+
+ if limit.hard == "unlimited" {
+ rLimit.Max = unix.RLIM_INFINITY
+ } else {
+ hardLimit, err := strconv.ParseUint(limit.hard, 10, 64)
+ if err != nil {
+ return fmt.Errorf("Invalid hard limit for %q", limit.name)
+ }
+ rLimit.Max = hardLimit
+ }
+
+ err := unix.Setrlimit(resource, &rLimit)
+ if err != nil {
+ return err
+ }
+ }
+
+ if len(cmdParts) == 0 {
+ cmd.Help()
+ return fmt.Errorf("Missing required command argument")
+ }
+
+ execCmd := exec.Command(cmdParts[0], cmdParts[1:]...)
+ execCmd.Stdin = os.Stdin
+ execCmd.Stdout = os.Stdout
+ execCmd.Stderr = os.Stderr
+
+ // Pass through any file descriptors specified.
+ for _, fd := range fds {
+ execCmd.ExtraFiles = append(execCmd.ExtraFiles, os.NewFile(fd, ""))
+ }
+
+ return execCmd.Run()
+}
From a03f7bc313ae2aef085456de04c1fe991937ad7e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parrott at canonical.com>
Date: Wed, 29 Jan 2020 16:06:59 +0000
Subject: [PATCH 3/3] lxd/instance/drivers/driver/qemu: Switches to launching
qemu via forklimits
This allows the memlock rlimit to be set to unlimited for PCI passthrough.
Signed-off-by: Thomas Parrott <thomas.parrott at canonical.com>
---
lxd/instance/drivers/driver_qemu.go | 25 +++++++++++++++++++------
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/lxd/instance/drivers/driver_qemu.go b/lxd/instance/drivers/driver_qemu.go
index 4ebdc409bf..de186751cc 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -690,7 +690,9 @@ func (vm *qemu) Start(stateful bool) error {
return err
}
- args := []string{
+ qemuCmd := []string{
+ "--",
+ qemuBinary,
"-S",
"-name", vm.Name(),
"-uuid", vmUUID,
@@ -709,7 +711,7 @@ func (vm *qemu) Start(stateful bool) error {
// Attempt to drop privileges.
if vm.state.OS.UnprivUser != "" {
- args = append(args, "-runas", vm.state.OS.UnprivUser)
+ qemuCmd = append(qemuCmd, "-runas", vm.state.OS.UnprivUser)
// Change ownership of config directory files so they are accessible to the
// unprivileged qemu process so that the 9p share can work.
@@ -740,15 +742,26 @@ func (vm *qemu) Start(stateful bool) error {
}
if shared.IsTrue(vm.expandedConfig["limits.memory.hugepages"]) {
- args = append(args, "-mem-path", "/dev/hugepages/", "-mem-prealloc")
+ qemuCmd = append(qemuCmd, "-mem-path", "/dev/hugepages/", "-mem-prealloc")
}
if vm.expandedConfig["raw.qemu"] != "" {
fields := strings.Split(vm.expandedConfig["raw.qemu"], " ")
- args = append(args, fields...)
+ qemuCmd = append(qemuCmd, fields...)
}
- cmd := exec.Command(qemuBinary, args...)
+ // Run the qemu command via forklimits so we can selectively increase ulimits.
+ forkLimitsCmd := []string{
+ "forklimits",
+ "limit=memlock:unlimited:unlimited", // Required for PCI passthrough.
+ }
+
+ for i := range fdFiles {
+ // Pass through any file descriptors as 3+i (as first 3 file descriptors are taken as standard).
+ forkLimitsCmd = append(forkLimitsCmd, fmt.Sprintf("fd=%d", 3+i))
+ }
+
+ cmd := exec.Command(vm.state.OS.ExecPath, append(forkLimitsCmd, qemuCmd...)...)
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &stdout
@@ -768,7 +781,7 @@ func (vm *qemu) Start(stateful bool) error {
err = cmd.Run()
if err != nil {
- err = errors.Wrapf(err, "Failed to run: %s %s: %s", qemuBinary, strings.Join(args, " "), strings.TrimSpace(string(stderr.Bytes())))
+ err = errors.Wrapf(err, "Failed to run: %s: %s", strings.Join(cmd.Args, " "), strings.TrimSpace(string(stderr.Bytes())))
op.Done(err)
return err
}
More information about the lxc-devel
mailing list