[lxc-devel] [lxd/master] Improve shutdown logic for cluster nodes

[monstermunchkin on Github]

A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 314 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20180801/3c3f2a12/attachment.bin>
-------------- next part --------------
From c205f2704d2342274a9fc0d4ff78c47dfe834b5d Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Mon, 30 Jul 2018 18:42:51 +0200
Subject: [PATCH 1/2] lxd: Improve shutdown logic for cluster nodes

Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
 lxd/containers.go | 38 ++++++++++++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/lxd/containers.go b/lxd/containers.go
index 726d0ee0f9..aa0da82a8d 100644
--- a/lxd/containers.go
+++ b/lxd/containers.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"io/ioutil"
 	"sort"
 	"strconv"
 	"sync"
@@ -196,16 +197,39 @@ func (slice containerStopList) Swap(i, j int) {
 func containersShutdown(s *state.State) error {
 	var wg sync.WaitGroup
 
+	dbAvailable := true
+
 	// Get all the containers
 	results, err := s.Cluster.ContainersList(db.CTypeRegular)
 	if err != nil {
-		return err
+		// Mark database as offline
+		dbAvailable = false
+
+		// List all containers on disk
+		files, err := ioutil.ReadDir(shared.VarPath("containers"))
+		if err != nil {
+			return err
+		}
+
+		for _, file := range files {
+			results = append(results, file.Name())
+		}
 	}
 
 	containers := []container{}
 
 	for _, name := range results {
-		c, err := containerLoadByName(s, name)
+		var c container
+		var err error
+
+		if dbAvailable {
+			c, err = containerLoadByName(s, name)
+		} else {
+			c, err = containerLXCLoad(s, db.ContainerArgs{
+				Name:   name,
+				Config: make(map[string]string),
+			})
+		}
 		if err != nil {
 			return err
 		}
@@ -215,10 +239,12 @@ func containersShutdown(s *state.State) error {
 
 	sort.Sort(containerStopList(containers))
 
-	// Reset all container states
-	err = s.Cluster.ContainersResetState()
-	if err != nil {
-		return err
+	if dbAvailable {
+		// Reset all container states
+		err = s.Cluster.ContainersResetState()
+		if err != nil {
+			return err
+		}
 	}
 
 	var lastPriority int = 0

From d259e0db569caf08a1e7102c9108773e344a2ae6 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Wed, 1 Aug 2018 16:29:53 +0200
Subject: [PATCH 2/2] test: Add test for cluster shutdown logic

Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
---
 test/suites/clustering.sh | 65 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/test/suites/clustering.sh b/test/suites/clustering.sh
index 28bd389134..f30bf0ce4b 100644
--- a/test/suites/clustering.sh
+++ b/test/suites/clustering.sh
@@ -903,3 +903,68 @@ test_clustering_join_api() {
   kill_lxd "${LXD_TWO_DIR}"
   kill_lxd "${LXD_ONE_DIR}"
 }
+
+test_clustering_shutdown_nodes() {
+  setup_clustering_bridge
+  prefix="lxd$$"
+  bridge="${prefix}"
+
+  setup_clustering_netns 1
+  LXD_ONE_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
+  chmod +x "${LXD_ONE_DIR}"
+  ns1="${prefix}1"
+  spawn_lxd_and_bootstrap_cluster "${ns1}" "${bridge}" "${LXD_ONE_DIR}"
+
+  # Add a newline at the end of each line. YAML as weird rules..
+  cert=$(sed ':a;N;$!ba;s/\n/\n\n/g' "${LXD_ONE_DIR}/server.crt")
+
+  # Spawn a second node
+  setup_clustering_netns 2
+  LXD_TWO_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
+  chmod +x "${LXD_TWO_DIR}"
+  ns2="${prefix}2"
+  spawn_lxd_and_join_cluster "${ns2}" "${bridge}" "${cert}" 2 1 "${LXD_TWO_DIR}"
+
+  # Spawn a third node
+  setup_clustering_netns 3
+  LXD_THREE_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
+  chmod +x "${LXD_THREE_DIR}"
+  ns3="${prefix}3"
+  spawn_lxd_and_join_cluster "${ns3}" "${bridge}" "${cert}" 3 1 "${LXD_THREE_DIR}"
+
+  # Init a container on node2, using a client connected to node1
+  LXD_DIR="${LXD_ONE_DIR}" ensure_import_testimage
+  LXD_DIR="${LXD_ONE_DIR}" lxc launch --target node1 testimage foo
+
+  # Get container PID
+  LXD_DIR="${LXD_ONE_DIR}" lxc info foo | grep Pid | cut -d' ' -f2 > foo.pid
+
+  # Get server PIDs
+  LXD_DIR="${LXD_ONE_DIR}" lxc info | awk '/server_pid/{print $2}' > one.pid
+  LXD_DIR="${LXD_TWO_DIR}" lxc info | awk '/server_pid/{print $2}' > two.pid
+  LXD_DIR="${LXD_THREE_DIR}" lxc info | awk '/server_pid/{print $2}' > three.pid
+
+  LXD_DIR="${LXD_TWO_DIR}" lxd shutdown
+  wait "$(cat two.pid)"
+  LXD_DIR="${LXD_THREE_DIR}" lxd shutdown
+  wait "$(cat three.pid)"
+  # Make sure the database is not available to the first node
+  sleep 5
+  LXD_DIR="${LXD_ONE_DIR}" lxd shutdown
+
+  # Wait for LXD to terminate, otherwise the db will not be empty, and the
+  # cleanup code will fail
+  wait "$(cat one.pid)"
+
+  # Container foo shouldn't be running anymore
+  ! ps -q "$(cat foo.pid)"
+
+  rm -f one.pid two.pid three.pid foo.pid
+
+  teardown_clustering_netns
+  teardown_clustering_bridge
+
+  kill_lxd "${LXD_TWO_DIR}"
+  kill_lxd "${LXD_ONE_DIR}"
+  kill_lxd "${LXD_THREE_DIR}"
+}