[lxc-devel] [lxd/master] Improve graceful shutdown

[stgraber on Github]

A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20201007/b9247510/attachment.bin>
-------------- next part --------------
From 218d76f7b5d515019717a4a645984d59ad1baaa2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 7 Oct 2020 22:26:14 -0400
Subject: [PATCH 1/2] lxd/daemon: Clean shutdown on SIGPWR/SIGTERM
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/main_daemon.go | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/lxd/main_daemon.go b/lxd/main_daemon.go
index 9cf3f7aad5..99ad98ea53 100644
--- a/lxd/main_daemon.go
+++ b/lxd/main_daemon.go
@@ -77,13 +77,30 @@ func (c *cmdDaemon) Run(cmd *cobra.Command, args []string) error {
 	signal.Notify(chIgnore, unix.SIGHUP)
 
 	s := d.State()
+
+	cleanStop := func() {
+		// Cancelling the context will make everyone aware that we're shutting down.
+		d.cancel()
+
+		// waitForOperations will block until all operations are done, or it's forced to shut down.
+		// For the latter case, we re-use the shutdown channel which is filled when a shutdown is
+		// initiated using `lxd shutdown`.
+		waitForOperations(s, d.shutdownChan)
+
+		d.Kill()
+	}
+
 	select {
 	case sig := <-ch:
 		if sig == unix.SIGPWR {
-			logger.Infof("Received '%s signal', shutting down instances", sig)
-			d.Kill()
+			logger.Infof("Received '%s signal', waiting for all operations to finish", sig)
+			cleanStop()
+
 			instancesShutdown(s)
 			networkShutdown(s)
+		} else if sig == unix.SIGTERM {
+			logger.Infof("Received '%s signal', waiting for all operations to finish", sig)
+			cleanStop()
 		} else {
 			logger.Infof("Received '%s signal', exiting", sig)
 			d.Kill()
@@ -91,14 +108,8 @@ func (c *cmdDaemon) Run(cmd *cobra.Command, args []string) error {
 
 	case <-d.shutdownChan:
 		logger.Infof("Asked to shutdown by API, waiting for all operations to finish")
-		// Cancelling the context will make everyone aware that we're shutting down.
-		d.cancel()
-		// waitForOperations will block until all operations are done, or it's forced to shut down.
-		// For the latter case, we re-use the shutdown channel which is filled when a shutdown is
-		// initiated using `lxd shutdown`.
-		waitForOperations(s, d.shutdownChan)
+		cleanStop()
 
-		d.Kill()
 		instancesShutdown(s)
 		networkShutdown(s)
 	}

From 33b7ecc2c230462ce42014bcb7a82c6e0887c09d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 7 Oct 2020 22:34:47 -0400
Subject: [PATCH 2/2] lxd/operations: Don't directly trigger shutdown
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/operations.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lxd/operations.go b/lxd/operations.go
index e1815a02ef..aa982b7034 100644
--- a/lxd/operations.go
+++ b/lxd/operations.go
@@ -95,8 +95,7 @@ func waitForOperations(s *state.State, chCancel chan struct{}) {
 			// If there are still running operations, we shut down the instances
 			// which will terminate the operations.
 			if execConsoleOps > 0 {
-				logger.Info("Shutdown timeout reached, shutting down instances")
-				instancesShutdown(s)
+				logger.Info("Timeout reached, continuing with shutdown")
 			}
 
 		case <-logTick: