[lxc-devel] [lxd/master] Fix raft_nodes/nodes ID disconnect

[stgraber on Github]

A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190718/75fbbccd/attachment-0001.bin>
-------------- next part --------------
From 3cce1bcd8d7fcfe87017429cb88bcf431728aedb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 18 Jul 2019 04:37:28 -0400
Subject: [PATCH 1/2] lxd/cluster/membership: Fix new DB server id
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This effectively reverts commit 1b52ef853533efb7e0837a1aae731f925043617b.

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/cluster/membership.go | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/lxd/cluster/membership.go b/lxd/cluster/membership.go
index 671cd4a886..8a1b186c1f 100644
--- a/lxd/cluster/membership.go
+++ b/lxd/cluster/membership.go
@@ -468,7 +468,6 @@ func Rebalance(state *state.State, gateway *Gateway) (string, []db.RaftNode, err
 
 	// Check if we have a spare node that we can turn into a database one.
 	address := ""
-	id := int64(-1)
 	err = state.Cluster.Transaction(func(tx *db.ClusterTx) error {
 		config, err := ConfigLoad(tx)
 		if err != nil {
@@ -489,7 +488,6 @@ func Rebalance(state *state.State, gateway *Gateway) (string, []db.RaftNode, err
 			logger.Debugf(
 				"Found spare node %s (%s) to be promoted as database node", node.Name, node.Address)
 			address = node.Address
-			id = node.ID
 			break
 		}
 
@@ -504,9 +502,20 @@ func Rebalance(state *state.State, gateway *Gateway) (string, []db.RaftNode, err
 		return "", currentRaftNodes, nil
 	}
 
-	// Update the local raft_table adding the new member and building a new
-	// list.
-	updatedRaftNodes := append(currentRaftNodes, db.RaftNode{ID: id, Address: address})
+	// Figure out the next ID in the raft_nodes table
+	var updatedRaftNodes []db.RaftNode
+	err = gateway.db.Transaction(func(tx *db.NodeTx) error {
+		id, err := tx.RaftNodeAdd(address)
+		if err != nil {
+			return errors.Wrap(err, "Failed to add new raft node")
+		}
+
+		updatedRaftNodes = append(currentRaftNodes, db.RaftNode{ID: id, Address: address})
+		return nil
+	})
+	if err != nil {
+		return "", nil, err
+	}
 
 	return address, updatedRaftNodes, nil
 }

From bcf52e7ef6e5c02ae4906d14db63948a701582e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 18 Jul 2019 04:57:14 -0400
Subject: [PATCH 2/2] lxd/cluster/heartbeat: Properly handle raft_nodes/nodes
 ID disconnect
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
 lxd/cluster/gateway.go   |  7 ++++++-
 lxd/cluster/heartbeat.go | 20 +++++++++++++++-----
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 78a9264a46..f14e873582 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -180,8 +180,13 @@ func (g *Gateway) HandlerFuncs(nodeRefreshTask func(*APIHeartbeat)) map[string]h
 			nodes := make([]db.RaftNode, 0)
 			for _, node := range heartbeatData.Members {
 				if node.Raft {
+					// Needed to handle LXD 3.15
+					if node.RaftID == 0 {
+						node.RaftID = node.ID
+					}
+
 					nodes = append(nodes, db.RaftNode{
-						ID:      node.ID,
+						ID:      node.RaftID,
 						Address: node.Address,
 					})
 				}
diff --git a/lxd/cluster/heartbeat.go b/lxd/cluster/heartbeat.go
index 45bca1d908..19219cd09c 100644
--- a/lxd/cluster/heartbeat.go
+++ b/lxd/cluster/heartbeat.go
@@ -22,6 +22,7 @@ type APIHeartbeatMember struct {
 	ID            int64
 	Address       string
 	Raft          bool
+	RaftID        int64
 	LastHeartbeat time.Time
 	Online        bool // Calculated from offline threshold and LastHeatbeat time.
 	updated       bool // Has node been updated during this heartbeat run. Not sent to nodes.
@@ -46,6 +47,16 @@ type APIHeartbeat struct {
 	FullStateList bool
 }
 
+func (hbState *APIHeartbeat) memberByAddress(address string) (APIHeartbeatMember, bool) {
+	for _, member := range hbState.Members {
+		if member.Address == address {
+			return member, true
+		}
+	}
+
+	return APIHeartbeatMember{}, false
+}
+
 // Update updates an existing APIHeartbeat struct with the raft and all node states supplied.
 // If allNodes provided is an empty set then this is considered a non-full state list.
 func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode, allNodes []db.NodeInfo, offlineThreshold time.Duration) {
@@ -61,16 +72,16 @@ func (hbState *APIHeartbeat) Update(fullStateList bool, raftNodes []db.RaftNode,
 
 	// Add raft nodes first with the raft flag set to true, but missing LastHeartbeat time.
 	for _, node := range raftNodes {
-		member, exists := hbState.Members[node.ID]
+		member, exists := hbState.memberByAddress(node.Address)
 		if !exists {
 			member = APIHeartbeatMember{
-				ID:      node.ID,
 				Address: node.Address,
 			}
 		}
 
+		member.RaftID = node.ID
 		member.Raft = true
-		hbState.Members[node.ID] = member
+		hbState.Members[member.ID] = member
 	}
 
 	// Add remaining nodes, and when if existing node is found, update status.
@@ -121,7 +132,6 @@ func (hbState *APIHeartbeat) Send(ctx context.Context, cert *shared.CertInfo, lo
 		logger.Debugf("Sending heartbeat to %s", address)
 
 		err := HeartbeatNode(ctx, address, cert, heartbeatData)
-
 		if err == nil {
 			hbState.Lock()
 			// Ensure only update nodes that exist in Members already.
@@ -281,7 +291,7 @@ func (g *Gateway) heartbeat(ctx context.Context, initialHeartbeat bool) {
 	for _, currentNode := range currentNodes {
 		existing := false
 		for _, node := range allNodes {
-			if node.Address == currentNode.Address && node.ID == currentNode.ID {
+			if node.Address == currentNode.Address {
 				existing = true
 				break
 			}