[lxc-devel] [lxd/master] Fix hangs on migration failure

[stgraber on Github]

A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190207/2e5321fe/attachment.bin>
-------------- next part --------------
From c14e2af31482215a6344461f2bfdd6e335f055be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 7 Feb 2019 16:17:56 -0500
Subject: [PATCH 1/3] lxd/migration: Fix race in abort
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/migrate_container.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lxd/migrate_container.go b/lxd/migrate_container.go
index 17ae799972..6c9b5ee908 100644
--- a/lxd/migrate_container.go
+++ b/lxd/migrate_container.go
@@ -474,7 +474,7 @@ func (s *migrationSourceWs) Do(migrateOp *operation) error {
 	// without introducing the fragility of closing on err.
 	abort := func(err error) error {
 		driver.Cleanup()
-		s.sendControl(err)
+		go s.sendControl(err)
 		return err
 	}
 

From 54eaf33dc826f55d2b707e4f268044713f71e181 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 7 Feb 2019 16:18:52 -0500
Subject: [PATCH 2/3] lxd/migration: Handle crashing rsync
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/rsync.go | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/lxd/rsync.go b/lxd/rsync.go
index c6e3388ab9..f8f2f63cbd 100644
--- a/lxd/rsync.go
+++ b/lxd/rsync.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/exec"
 	"syscall"
+	"time"
 
 	"github.com/gorilla/websocket"
 	"github.com/pborman/uuid"
@@ -87,6 +88,7 @@ func rsyncSendSetup(name string, path string, bwlimit string, execPath string, f
 	if err != nil {
 		return nil, nil, nil, err
 	}
+	defer l.Close()
 
 	/*
 	 * Here, the path /tmp/foo is ignored. Since we specify localhost,
@@ -135,15 +137,34 @@ func rsyncSendSetup(name string, path string, bwlimit string, execPath string, f
 		return nil, nil, nil, err
 	}
 
-	conn, err := l.Accept()
-	if err != nil {
+	var conn *net.Conn
+	chConn := make(chan *net.Conn, 1)
+
+	go func() {
+		conn, err := l.Accept()
+		if err != nil {
+			chConn <- nil
+			return
+		}
+
+		chConn <- &conn
+	}()
+
+	select {
+	case conn = <-chConn:
+		if conn == nil {
+			cmd.Process.Kill()
+			cmd.Wait()
+			return nil, nil, nil, fmt.Errorf("Failed to connect to rsync socket")
+		}
+
+	case <-time.After(10 * time.Second):
 		cmd.Process.Kill()
 		cmd.Wait()
-		return nil, nil, nil, err
+		return nil, nil, nil, fmt.Errorf("rsync failed to spawn after 10s")
 	}
-	l.Close()
 
-	return cmd, conn, stderr, nil
+	return cmd, *conn, stderr, nil
 }
 
 // RsyncSend sets up the sending half of an rsync, to recursively send the

From e2c9f10e8b528a1029e77f49ab46f0efa09bb376 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 7 Feb 2019 16:18:59 -0500
Subject: [PATCH 3/3] lxd/migration: Fix sender side errors handling
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Closes #5445

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/rsync.go | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/lxd/rsync.go b/lxd/rsync.go
index f8f2f63cbd..e4af344557 100644
--- a/lxd/rsync.go
+++ b/lxd/rsync.go
@@ -186,14 +186,22 @@ func RsyncSend(name string, path string, conn *websocket.Conn, readWrapper func(
 
 	readDone, writeDone := shared.WebsocketMirror(conn, dataSocket, readPipe, nil, nil)
 
+	chError := make(chan error, 1)
+	go func() {
+		err = cmd.Wait()
+		if err != nil {
+			dataSocket.Close()
+			readPipe.Close()
+		}
+		chError <- err
+	}()
+
 	output, err := ioutil.ReadAll(stderr)
 	if err != nil {
 		cmd.Process.Kill()
-		cmd.Wait()
-		return err
 	}
 
-	err = cmd.Wait()
+	err = <-chError
 	if err != nil {
 		logger.Errorf("Rsync send failed: %s: %s: %s", path, err, string(output))
 	}