[lxc-devel] [lxd/master] implement and switch to ExecNoWait()
brauner on Github
lxc-bot at linuxcontainers.org
Wed Oct 12 16:53:41 UTC 2016
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 2951 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20161012/5866fd06/attachment.bin>
-------------- next part --------------
From cc9c20dc4e12b023584e1ea7d8638ccf0cb62e3e Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at canonical.com>
Date: Tue, 11 Oct 2016 15:04:25 +0200
Subject: [PATCH 1/5] lxd/container_lxc: add prepareExec() helper fun
We will add another Exec*() style function that does the same setup operations
as Exec(). So let's create a common helper that does the setup for both.
Signed-off-by: Christian Brauner <christian.brauner at canonical.com>
---
lxd/container_lxc.go | 24 +++++++++++++++++++-----
1 file changed, 19 insertions(+), 5 deletions(-)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index e67eacc..703e9f5 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -3633,14 +3633,24 @@ func (c *containerLXC) FilePush(srcpath string, dstpath string, uid int, gid int
return nil
}
-func (c *containerLXC) Exec(command []string, env map[string]string, stdin *os.File, stdout *os.File, stderr *os.File) (int, error) {
+func (c *containerLXC) prepareExec(waitOnPid bool, command []string, env map[string]string, stdin *os.File, stdout *os.File, stderr *os.File) ([]string, *exec.Cmd) {
envSlice := []string{}
for k, v := range env {
envSlice = append(envSlice, fmt.Sprintf("%s=%s", k, v))
}
- args := []string{execPath, "forkexec", c.name, c.daemon.lxcpath, filepath.Join(c.LogPath(), "lxc.conf")}
+ // Setting "wait"/"nowait" as the second argument in cmd.Args allows us
+ // to tell execContainer() that it should call an appropriate go-lxc
+ // wrapper for c->attach() that executes the command we requested in the
+ // container and, depending on whether we passed "wait" or "nowait" does
+ // wait or not wait for it to exit.
+ var args []string
+ if waitOnPid {
+ args = []string{execPath, "forkexec", "wait", c.name, c.daemon.lxcpath, filepath.Join(c.LogPath(), "lxc.conf")}
+ } else {
+ args = []string{execPath, "forkexec", "nowait", c.name, c.daemon.lxcpath, filepath.Join(c.LogPath(), "lxc.conf")}
+ }
args = append(args, "--")
args = append(args, "env")
@@ -3658,23 +3668,27 @@ func (c *containerLXC) Exec(command []string, env map[string]string, stdin *os.F
cmd.Stderr = stderr
shared.LogInfo("Executing command", log.Ctx{"environment": envSlice, "args": args})
+ return envSlice, &cmd
+}
+func (c *containerLXC) Exec(command []string, env map[string]string, stdin *os.File, stdout *os.File, stderr *os.File) (int, error) {
+ envSlice, cmd := c.prepareExec(true, command, env, stdin, stdout, stderr)
err := cmd.Run()
if err != nil {
exitErr, ok := err.(*exec.ExitError)
if ok {
status, ok := exitErr.Sys().(syscall.WaitStatus)
if ok {
- shared.LogInfo("Executed command", log.Ctx{"environment": envSlice, "args": args, "exit_status": status.ExitStatus()})
+ shared.LogInfo("Executed command", log.Ctx{"environment": envSlice, "args": cmd.Args, "exit_status": status.ExitStatus()})
return status.ExitStatus(), nil
}
}
- shared.LogInfo("Failed executing command", log.Ctx{"environment": envSlice, "args": args, "err": err})
+ shared.LogInfo("Failed executing command", log.Ctx{"environment": envSlice, "args": cmd.Args, "err": err})
return -1, err
}
- shared.LogInfo("Executed command", log.Ctx{"environment": envSlice, "args": args})
+ shared.LogInfo("Executed command", log.Ctx{"environment": envSlice, "args": cmd.Args})
return 0, nil
}
From ae16e7035408d1fdc3d576239be5305ad07202c4 Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at canonical.com>
Date: Wed, 12 Oct 2016 13:00:12 +0200
Subject: [PATCH 2/5] lxd/container_lxc, lxd/container: add ExecNoWait()
ExecNoWait() constructs a lxd forkexec command that calls a go-lxc binding which
in turn calls low-level liblxc's c->attach(). In contrast to Exec() ExecNoWait()
calls a go-lxc wraper that does not wait on the command to exit. Rather,
the go-lxc wrapper will simply return the PID it receives from low-level
liblxc's c->attach(). Hence, it returns the PID of the attached process. To
retrieve the attached PID from the go-lxc wrapper ExecNoWait() passes a pipe fd
to lxd forkexec which will allow it to write the PID of the attached process to
the pipe so we can retrieve it ExecNoWait() again.
Signed-off-by: Christian Brauner <christian.brauner at canonical.com>
---
lxd/container.go | 6 +++++-
lxd/container_lxc.go | 23 +++++++++++++++++++++++
2 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/lxd/container.go b/lxd/container.go
index 6e0150c..4acbe3c 100644
--- a/lxd/container.go
+++ b/lxd/container.go
@@ -328,9 +328,13 @@ type container interface {
FilePull(srcpath string, dstpath string) (int, int, os.FileMode, string, []string, error)
FilePush(srcpath string, dstpath string, uid int, gid int, mode int) error
- // Command execution
+ // Command execution: Execute command and wait for it to exit.
Exec(command []string, env map[string]string, stdin *os.File, stdout *os.File, stderr *os.File) (int, error)
+ // Command execution: Execute command and return PID of executing
+ // process. Does not wait for the process to exit.
+ ExecNoWait(command []string, env map[string]string, stdin *os.File, stdout *os.File, stderr *os.File, attachedPid *int) (int, error)
+
// Status
Render() (interface{}, interface{}, error)
RenderState() (*shared.ContainerState, error)
diff --git a/lxd/container_lxc.go b/lxd/container_lxc.go
index 703e9f5..82675ba 100644
--- a/lxd/container_lxc.go
+++ b/lxd/container_lxc.go
@@ -3692,6 +3692,29 @@ func (c *containerLXC) Exec(command []string, env map[string]string, stdin *os.F
return 0, nil
}
+func (c *containerLXC) ExecNoWait(command []string, env map[string]string, stdin *os.File, stdout *os.File, stderr *os.File, attachedPid *int) (int, error) {
+ envSlice, cmd := c.prepareExec(false, command, env, stdin, stdout, stderr)
+ r, w, err := os.Pipe()
+ defer r.Close()
+ defer w.Close()
+ if err != nil {
+ shared.LogErrorf("Failed to create pipe to pass to child..")
+ return -1, err
+ }
+ cmd.ExtraFiles = []*os.File{w}
+ err = cmd.Run()
+ if err != nil {
+ shared.LogInfo("Failed executing command", log.Ctx{"environment": envSlice, "args": cmd.Args, "err": err})
+ return -1, err
+ }
+
+ if err := json.NewDecoder(r).Decode(&attachedPid); err != nil {
+ shared.LogErrorf("Failed to retrieve PID of executing child process.")
+ return -1, err
+ }
+ return 0, nil
+}
+
func (c *containerLXC) cpuState() shared.ContainerStateCPU {
cpu := shared.ContainerStateCPU{}
From f012270435e8d734d35b2d7ae42a86e84ad4e6c7 Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at canonical.com>
Date: Wed, 12 Oct 2016 13:45:58 +0200
Subject: [PATCH 3/5] shared/util_linux: add Prctl() and SetSubReaper()
- Add Prctl() to allow various operations on processes.
- Add SetSubReaper() to mark a process as subreaper.
Signed-off-by: Christian Brauner <christian.brauner at canonical.com>
---
shared/util_linux.go | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/shared/util_linux.go b/shared/util_linux.go
index 51e0e43..c1b6ec4 100644
--- a/shared/util_linux.go
+++ b/shared/util_linux.go
@@ -482,3 +482,25 @@ func GetAllXattr(path string) (xattrs map[string]string, err error) {
return xattrs, nil
}
+
+// Perform various operations on a process.
+func Prctl(option int, arg2, arg3, arg4, arg5 uintptr) error {
+ var err error
+ _, _, errno := syscall.Syscall6(syscall.SYS_PRCTL, uintptr(option), arg2, arg3, arg4, arg5, 0)
+ if errno != 0 {
+ err = errno
+ }
+ return err
+}
+
+// Mark a process as a child subreaper. This allows the process to act as init
+// for all children it creates, and their descendants. If the immediate parent
+// of a grandchild has already terminated, the nearest still living ancestor
+// subreaper will receive a SIGCHLD signal and will be able to wait(2) on the
+// process.
+const PR_SET_CHILD_SUBREAPER = 36
+
+// Mark a process as a child subreaper.
+func SetSubReaper(val int) error {
+ return Prctl(PR_SET_CHILD_SUBREAPER, uintptr(val), 0, 0, 0)
+}
From 5fa3244535eea4ba8e046ac0fcd94e688734e0e3 Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at canonical.com>
Date: Wed, 12 Oct 2016 13:53:37 +0200
Subject: [PATCH 4/5] lxd/containers: call RunCommandNoWait()
When lxd forkexec is called as
lxd forkexec wait ...
then we call
RunCommandStatus()
when lxd forkexec is called as
lxd forkexec nowait ...
then we call
RunCommandNoWait()
The difference is that RunCommandStatus() is a wrapper that runs and waits for
the command to exit whereas RunCommandNoWait() runs the command, returns its PID
but does not wait for it to exit.
Signed-off-by: Christian Brauner <christian.brauner at canonical.com>
---
lxd/containers.go | 34 ++++++++++++++++++++++++++++------
1 file changed, 28 insertions(+), 6 deletions(-)
diff --git a/lxd/containers.go b/lxd/containers.go
index 3449d69..7fee727 100644
--- a/lxd/containers.go
+++ b/lxd/containers.go
@@ -1,6 +1,7 @@
package main
import (
+ "encoding/json"
"fmt"
"os"
"sort"
@@ -281,9 +282,13 @@ func execContainer(args []string) (int, error) {
return -1, fmt.Errorf("Bad arguments: %q", args)
}
- name := args[1]
- lxcpath := args[2]
- configPath := args[3]
+ wait := true
+ if args[1] == "nowait" {
+ wait = false
+ }
+ name := args[2]
+ lxcpath := args[3]
+ configPath := args[4]
c, err := lxc.NewContainer(name, lxcpath)
if err != nil {
@@ -351,9 +356,26 @@ func execContainer(args []string) (int, error) {
opts.Env = env
- status, err := c.RunCommandStatus(cmd, opts)
- if err != nil {
- return -1, fmt.Errorf("Failed running command: %q", err)
+ var status int
+ if wait {
+ status, err = c.RunCommandStatus(cmd, opts)
+ if err != nil {
+ return -1, fmt.Errorf("Failed running command and waiting for it to exit: %q", err)
+ }
+ } else {
+ status, err = c.RunCommandNoWait(cmd, opts)
+ if err != nil {
+ return -1, fmt.Errorf("Failed running command: %q", err)
+ }
+ // Send the PID of the executing process.
+ w := os.NewFile(uintptr(3), "attachedPid")
+ defer w.Close()
+
+ err = json.NewEncoder(w).Encode(status)
+ if err != nil {
+ return -1, fmt.Errorf("Failed sending PID of executing command: %q", err)
+ }
+ return 0, nil
}
return status >> 8, nil
From dc9dd15250efc527476f4e0087a9b75e5375cce4 Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at canonical.com>
Date: Wed, 12 Oct 2016 14:43:53 +0200
Subject: [PATCH 5/5] lxd/container_exec: switch to ExecNoWait()
In order to forward signals to the program we are executing we need a function
that hands us the PID of the executing process. After ExecNoWait() has been
called we want to be able to wait on the process whose PID we return in
attachedPid. But since this process is a child of lxd forkexec the caller cannot
directly wait on it. Luckily, in the case of ExecNoWait() lxd forkexec will
call a go-lxc binding that calls plain c->attach() which only starts the process
and the immediately exits. Which in turn causes lxd forkexec to immediately
exit. So in order for the caller of ExecNoWait() to be able to wait on said
returned PID we can simply mark us as the subreaper for forkexec and its
descendants. The code tries to ensure that we are only a temporary subreaper,
i.e. we mark us as subreaper before the command and unset us as subreaper before
we return.
Signed-off-by: Christian Brauner <christian.brauner at canonical.com>
---
lxd/container_exec.go | 82 ++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 64 insertions(+), 18 deletions(-)
diff --git a/lxd/container_exec.go b/lxd/container_exec.go
index f782634..654ece7 100644
--- a/lxd/container_exec.go
+++ b/lxd/container_exec.go
@@ -10,6 +10,7 @@ import (
"strconv"
"strings"
"sync"
+ "syscall"
"github.com/gorilla/mux"
"github.com/gorilla/websocket"
@@ -216,33 +217,78 @@ func (s *execWs) Do(op *operation) error {
}
}
- cmdResult, cmdErr := s.container.Exec(s.command, s.env, stdin, stdout, stderr)
+ finisher := func(cmdResult int, cmdErr error, unsetSubreaper bool) error {
+ for _, tty := range ttys {
+ tty.Close()
+ }
- for _, tty := range ttys {
- tty.Close()
- }
+ if s.conns[-1] == nil {
+ if s.interactive {
+ controlExit <- true
+ }
+ } else {
+ s.conns[-1].Close()
+ }
+
+ wgEOF.Wait()
- if s.conns[-1] == nil {
- if s.interactive {
- controlExit <- true
+ for _, pty := range ptys {
+ pty.Close()
}
- } else {
- s.conns[-1].Close()
- }
- wgEOF.Wait()
+ if unsetSubreaper {
+ if err := shared.SetSubReaper(0); err != nil {
+ shared.LogDebugf("Unsetting LXD as temporary subreaper failed. prctl() returned: %s", err)
+ }
+ }
- for _, pty := range ptys {
- pty.Close()
+ metadata := shared.Jmap{"return": cmdResult}
+ err = op.UpdateMetadata(metadata)
+ if err != nil {
+ return err
+ }
+
+ return cmdErr
}
- metadata := shared.Jmap{"return": cmdResult}
- err = op.UpdateMetadata(metadata)
- if err != nil {
- return err
+ /* After ExecNoWait() has been called we want to be able to wait on the
+ * process whose PID we return in attachedPid. But since this process is
+ * a child of lxd forkexec the caller cannot directly wait on it.
+ * Luckily, in the case of ExecNoWait() lxd forkexec will call a go-lxc
+ * binding that calls plain c->attach() which only starts the process
+ * and the immediately exits. Which in turn causes lxd forkexec to
+ * immediately exit. So in order for the caller of ExecNoWait() to be
+ * able to wait on said returned PID we can simply mark us as the
+ * subreaper for forkexec and its descendants.
+ */
+ weAreSubreaper := true
+ if err := shared.SetSubReaper(1); err != nil {
+ // TODO: Not sure what's best if we fail to become subreaper.
+ // Should we fail immediately?
+ shared.LogDebug("Setting LXD as temporary subreaper failed. prctl() returned: %s", err)
+ weAreSubreaper = false
+ }
+
+ var attachedPid int
+ cmdResult, cmdErr := s.container.ExecNoWait(s.command, s.env, stdin, stdout, stderr, &attachedPid)
+ if cmdErr == nil {
+ proc, err := os.FindProcess(attachedPid)
+ if err != nil {
+ return finisher(cmdResult, err, weAreSubreaper)
+ }
+
+ procState, err := proc.Wait()
+ if err != nil {
+ return finisher(cmdResult, err, weAreSubreaper)
+ }
+
+ status, ok := procState.Sys().(syscall.WaitStatus)
+ if ok {
+ cmdResult = status.ExitStatus()
+ }
}
- return cmdErr
+ return finisher(cmdResult, cmdErr, weAreSubreaper)
}
func containerExecPost(d *Daemon, r *http.Request) Response {
More information about the lxc-devel
mailing list