[lxc-devel] [lxd/master] Remove raft node

freeekanayaka on Github lxc-bot at linuxcontainers.org
Mon Apr 6 10:30:45 UTC 2020


A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 318 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200406/7e7f07e8/attachment-0001.bin>
-------------- next part --------------
From 98894fe289541249e9c5775df114a6664ec485c2 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Mon, 6 Apr 2020 10:56:00 +0100
Subject: [PATCH 1/5] lxd/cluster: add RemoveRaftNode() to force removing a
 raft node

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 lxd/cluster/recover.go | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/lxd/cluster/recover.go b/lxd/cluster/recover.go
index 780feb5a1c..b41eb3cf62 100644
--- a/lxd/cluster/recover.go
+++ b/lxd/cluster/recover.go
@@ -1,10 +1,13 @@
 package cluster
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
+	"time"
 
 	dqlite "github.com/canonical/go-dqlite"
+	client "github.com/canonical/go-dqlite/client"
 	"github.com/lxc/lxd/lxd/db"
 	"github.com/lxc/lxd/lxd/node"
 	"github.com/pkg/errors"
@@ -87,3 +90,38 @@ func Recover(database *db.Node) error {
 
 	return nil
 }
+
+// RemoveRaftNode removes a raft node from the raft configuration.
+func RemoveRaftNode(gateway *Gateway, address string) error {
+	nodes, err := gateway.currentRaftNodes()
+	if err != nil {
+		return errors.Wrap(err, "Failed to get current raft nodes")
+	}
+	var id uint64
+	for _, node := range nodes {
+		if node.Address == address {
+			id = node.ID
+			break
+		}
+	}
+	if id == 0 {
+		return fmt.Errorf("No raft node with address %q", address)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	client, err := client.FindLeader(
+		ctx, gateway.NodeStore(),
+		client.WithDialFunc(gateway.raftDial()),
+		client.WithLogFunc(DqliteLog),
+	)
+	if err != nil {
+		return errors.Wrap(err, "Failed to connect to cluster leader")
+	}
+	defer client.Close()
+	err = client.Remove(ctx, id)
+	if err != nil {
+		return errors.Wrap(err, "Failed to remove node")
+	}
+	return nil
+}

From 85db63806a239ca011ede24d6d81641ad46a27d6 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Mon, 6 Apr 2020 10:56:23 +0100
Subject: [PATCH 2/5] api: Add "DELETE /internal/cluster/raft/<address>"
 endpoint

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 lxd/api_cluster.go  | 17 +++++++++++++++++
 lxd/api_internal.go |  1 +
 2 files changed, 18 insertions(+)

diff --git a/lxd/api_cluster.go b/lxd/api_cluster.go
index f9db3dc5da..5a812856e5 100644
--- a/lxd/api_cluster.go
+++ b/lxd/api_cluster.go
@@ -76,6 +76,12 @@ var internalClusterHandoverCmd = APIEndpoint{
 	Post: APIEndpointAction{Handler: internalClusterPostHandover},
 }
 
+var internalClusterRaftNodeCmd = APIEndpoint{
+	Path: "cluster/raft-node/{address}",
+
+	Delete: APIEndpointAction{Handler: internalClusterRaftNodeDelete},
+}
+
 // Return information about the cluster.
 func clusterGet(d *Daemon, r *http.Request) response.Response {
 	name := ""
@@ -1491,3 +1497,14 @@ func clusterCheckNetworksMatch(cluster *db.Cluster, reqNetworks []api.Network) e
 	}
 	return nil
 }
+
+// Used as low-level recovering helper.
+func internalClusterRaftNodeDelete(d *Daemon, r *http.Request) response.Response {
+	address := mux.Vars(r)["address"]
+	err := cluster.RemoveRaftNode(d.gateway, address)
+	if err != nil {
+		return response.SmartError(err)
+	}
+
+	return response.SyncResponse(true, nil)
+}
diff --git a/lxd/api_internal.go b/lxd/api_internal.go
index 46873aeb52..c5f1def35d 100644
--- a/lxd/api_internal.go
+++ b/lxd/api_internal.go
@@ -49,6 +49,7 @@ var apiInternal = []APIEndpoint{
 	internalGarbageCollectorCmd,
 	internalRAFTSnapshotCmd,
 	internalClusterHandoverCmd,
+	internalClusterRaftNodeCmd,
 }
 
 var internalShutdownCmd = APIEndpoint{

From a2ec082e11f788dbd6ce91ebd4836e68099edba8 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Mon, 6 Apr 2020 10:57:17 +0100
Subject: [PATCH 3/5] lxd: Add "lxd cluster remove-raft-node" recovery command

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 lxd/main_cluster.go | 71 ++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 70 insertions(+), 1 deletion(-)

diff --git a/lxd/main_cluster.go b/lxd/main_cluster.go
index 228b144a98..f9341b10bd 100644
--- a/lxd/main_cluster.go
+++ b/lxd/main_cluster.go
@@ -12,6 +12,7 @@ import (
 	"github.com/lxc/lxd/lxd/cluster"
 	"github.com/lxc/lxd/lxd/db"
 	"github.com/lxc/lxd/lxd/sys"
+	"github.com/lxc/lxd/lxd/util"
 	"github.com/lxc/lxd/shared"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -36,6 +37,10 @@ func (c *cmdCluster) Command() *cobra.Command {
 	recover := cmdClusterRecoverFromQuorumLoss{global: c.global}
 	cmd.AddCommand(recover.Command())
 
+	// Remove a raft node.
+	removeRaftNode := cmdClusterRemoveRaftNode{global: c.global}
+	cmd.AddCommand(removeRaftNode.Command())
+
 	return cmd
 }
 
@@ -102,7 +107,7 @@ func (c *cmdClusterRecoverFromQuorumLoss) Run(cmd *cobra.Command, args []string)
 		return fmt.Errorf("The LXD daemon is running, please stop it first.")
 	}
 
-	// Prompt for confiromation unless --quiet was passed.
+	// Prompt for confirmation unless --quiet was passed.
 	if !c.flagNonInteractive {
 		err := c.promptConfirmation()
 		if err != nil {
@@ -147,3 +152,67 @@ Do you want to proceed? (yes/no): `)
 	}
 	return nil
 }
+
+type cmdClusterRemoveRaftNode struct {
+	global             *cmdGlobal
+	flagNonInteractive bool
+}
+
+func (c *cmdClusterRemoveRaftNode) Command() *cobra.Command {
+	cmd := &cobra.Command{}
+	cmd.Use = "remove-raft-node <address>"
+	cmd.Aliases = []string{"ls"}
+	cmd.Short = "Remove a raft node from the raft configuration"
+
+	cmd.RunE = c.Run
+
+	cmd.Flags().BoolVarP(&c.flagNonInteractive, "quiet", "q", false, "Don't require user confirmation")
+
+	return cmd
+}
+
+func (c *cmdClusterRemoveRaftNode) Run(cmd *cobra.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Help()
+		return fmt.Errorf("Missing required arguments")
+	}
+
+	address := util.CanonicalNetworkAddress(args[0])
+
+	// Prompt for confirmation unless --quiet was passed.
+	if !c.flagNonInteractive {
+		err := c.promptConfirmation()
+		if err != nil {
+			return err
+		}
+	}
+
+	client, err := lxd.ConnectLXDUnix("", nil)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to connect to LXD daemon")
+	}
+
+	endpoint := fmt.Sprintf("/internal/cluster/raft-node/%s", address)
+	_, _, err = client.RawQuery("DELETE", endpoint, nil, "")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *cmdClusterRemoveRaftNode) promptConfirmation() error {
+	reader := bufio.NewReader(os.Stdin)
+	fmt.Printf(`You should run this command only if you ended up in an
+inconsistent state where a node has been uncleanly removed (i.e. it doesn't show
+up in "lxc cluster list" but it's still in the raft configuration).
+
+Do you want to proceed? (yes/no): `)
+	input, _ := reader.ReadString('\n')
+	input = strings.TrimSuffix(input, "\n")
+
+	if !shared.StringInSlice(strings.ToLower(input), []string{"yes"}) {
+		return fmt.Errorf("Remove raft node operation aborted")
+	}
+	return nil
+}

From ee4329a2c70447def9265eefc6ef5b999351dfd0 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Mon, 6 Apr 2020 11:29:12 +0100
Subject: [PATCH 4/5] doc: Add paragraph about "lxd cluster remove-raft-node"

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 doc/clustering.md | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/doc/clustering.md b/doc/clustering.md
index e9ee9d5605..cb90f4ed55 100644
--- a/doc/clustering.md
+++ b/doc/clustering.md
@@ -217,7 +217,7 @@ transition to the Blocked state, until you upgrade the very last
 one. At that point the blocked nodes will notice that there is no
 out-of-date node left and will become operational again.
 
-### Disaster recovery
+### Recover from quorum loss
 
 Every LXD cluster has up to 3 members that serve as database nodes. If you
 permanently lose a majority of the cluster members that are serving as database
@@ -294,6 +294,24 @@ lxc delete bionic
 lxc pull file bionic/etc/hosts .
 ```
 
+### Manually altering Raft membership
+
+There might be situations in which you need to manually alter the Raft
+membership configuration of the cluster because some unexpected behavior
+occurred.
+
+For example if you have a cluster member that was removed uncleanly it might not
+show up in `lxc cluster list` but still be part of the Raft configuration (you
+can see that with `lxd sql local "SELECT * FROM raft_nodes").
+
+In that case you can run:
+
+```bash
+lxd cluster remove-raft-node <address>
+```
+
+to remove the leftover node.
+
 ## Images
 
 By default, LXD will replicate images on as many cluster members as you

From b1df3fb8e5402b4f2a8f7f3e07b71a538152e8f0 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Mon, 6 Apr 2020 11:29:36 +0100
Subject: [PATCH 5/5] test: Add test exercising "lxd cluster remove-raft-node"

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 test/suites/clustering.sh | 106 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 106 insertions(+)

diff --git a/test/suites/clustering.sh b/test/suites/clustering.sh
index ab4e35714e..801cdc33a2 100644
--- a/test/suites/clustering.sh
+++ b/test/suites/clustering.sh
@@ -1695,3 +1695,109 @@ test_clustering_rebalance() {
   kill_lxd "${LXD_THREE_DIR}"
   kill_lxd "${LXD_FOUR_DIR}"
 }
+
+# Recover a cluster where a raft node was removed from the nodes table but not
+# from the raft configuration.
+test_clustering_remove_raft_node() {
+  # shellcheck disable=2039
+  local LXD_DIR
+
+  setup_clustering_bridge
+  prefix="lxd$$"
+  bridge="${prefix}"
+
+  setup_clustering_netns 1
+  LXD_ONE_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
+  chmod +x "${LXD_ONE_DIR}"
+  ns1="${prefix}1"
+  spawn_lxd_and_bootstrap_cluster "${ns1}" "${bridge}" "${LXD_ONE_DIR}"
+
+  # Add a newline at the end of each line. YAML as weird rules..
+  cert=$(sed ':a;N;$!ba;s/\n/\n\n/g' "${LXD_ONE_DIR}/server.crt")
+
+  # Spawn a second node
+  setup_clustering_netns 2
+  LXD_TWO_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
+  chmod +x "${LXD_TWO_DIR}"
+  ns2="${prefix}2"
+  spawn_lxd_and_join_cluster "${ns2}" "${bridge}" "${cert}" 2 1 "${LXD_TWO_DIR}"
+
+  # Configuration keys can be changed on any node.
+  LXD_DIR="${LXD_TWO_DIR}" lxc config set cluster.offline_threshold 40
+  LXD_DIR="${LXD_ONE_DIR}" lxc info | grep -q 'cluster.offline_threshold: "40"'
+  LXD_DIR="${LXD_TWO_DIR}" lxc info | grep -q 'cluster.offline_threshold: "40"'
+
+  # The preseeded network bridge exists on all nodes.
+  ns1_pid="$(cat "${TEST_DIR}/ns/${ns1}/PID")"
+  ns2_pid="$(cat "${TEST_DIR}/ns/${ns2}/PID")"
+  nsenter -m -n -t "${ns1_pid}" -- ip link show "${bridge}" > /dev/null
+  nsenter -m -n -t "${ns2_pid}" -- ip link show "${bridge}" > /dev/null
+
+  # Create a pending network and pool, to show that they are not
+  # considered when checking if the joining node has all the required
+  # networks and pools.
+  LXD_DIR="${LXD_TWO_DIR}" lxc storage create pool1 dir --target node1
+  LXD_DIR="${LXD_ONE_DIR}" lxc network create net1 --target node2
+
+  # Spawn a third node, using the non-leader node2 as join target.
+  setup_clustering_netns 3
+  LXD_THREE_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
+  chmod +x "${LXD_THREE_DIR}"
+  ns3="${prefix}3"
+  spawn_lxd_and_join_cluster "${ns3}" "${bridge}" "${cert}" 3 2 "${LXD_THREE_DIR}"
+
+  # Spawn a fourth node, this will be a non-database node.
+  setup_clustering_netns 4
+  LXD_FOUR_DIR=$(mktemp -d -p "${TEST_DIR}" XXX)
+  chmod +x "${LXD_FOUR_DIR}"
+  ns4="${prefix}4"
+  spawn_lxd_and_join_cluster "${ns4}" "${bridge}" "${cert}" 4 1 "${LXD_FOUR_DIR}"
+
+  # Kill the second node, to prevent it from transferring its database role at shutdown.
+  kill -9 "$(cat "${LXD_TWO_DIR}/lxd.pid")"
+
+  # Remove the second node from the database but not from the raft configuration.
+  LXD_DIR="${LXD_ONE_DIR}" lxd sql global "DELETE FROM nodes WHERE address = '10.1.1.102:8443'"
+
+  # The node does not appear anymore in the cluster list.
+  ! LXD_DIR="${LXD_ONE_DIR}" lxc cluster list | grep -q "node2" || false
+
+  # There are only 2 database nodes.
+  LXD_DIR="${LXD_ONE_DIR}" lxc cluster list | grep "node1" | grep -q "YES"
+  LXD_DIR="${LXD_ONE_DIR}" lxc cluster list | grep "node3" | grep -q "YES"
+  LXD_DIR="${LXD_ONE_DIR}" lxc cluster list | grep "node4" | grep -q "NO"
+
+  # The second node is still in the raft_nodes table.
+  LXD_DIR="${LXD_ONE_DIR}" lxd sql local "SELECT * FROM raft_nodes" | grep -q "10.1.1.102"
+
+  # Force removing the raft node.
+  LXD_DIR="${LXD_ONE_DIR}" lxd cluster remove-raft-node -q "10.1.1.102"
+
+  # Wait for a heartbeat to propagate.
+  sleep 15
+
+  # We're back to 3 database nodes.
+  LXD_DIR="${LXD_ONE_DIR}" lxc cluster list | grep "node1" | grep -q "YES"
+  LXD_DIR="${LXD_ONE_DIR}" lxc cluster list | grep "node3" | grep -q "YES"
+  LXD_DIR="${LXD_ONE_DIR}" lxc cluster list | grep "node4" | grep -q "YES"
+
+  # The second node is gone from the raft_nodes_table.
+  ! LXD_DIR="${LXD_ONE_DIR}" lxd sql local "SELECT * FROM raft_nodes" | grep -q "10.1.1.102" || false
+
+  LXD_DIR="${LXD_ONE_DIR}" lxd shutdown
+  LXD_DIR="${LXD_THREE_DIR}" lxd shutdown
+  LXD_DIR="${LXD_FOUR_DIR}" lxd shutdown
+  sleep 0.5
+  rm -f "${LXD_ONE_DIR}/unix.socket"
+  rm -f "${LXD_TWO_DIR}/unix.socket"
+  rm -f "${LXD_THREE_DIR}/unix.socket"
+  rm -f "${LXD_FOUR_DIR}/unix.socket"
+
+  teardown_clustering_netns
+  teardown_clustering_bridge
+
+  kill_lxd "${LXD_ONE_DIR}"
+  kill_lxd "${LXD_TWO_DIR}"
+  kill_lxd "${LXD_THREE_DIR}"
+  kill_lxd "${LXD_FOUR_DIR}"
+}


More information about the lxc-devel mailing list