[lxc-devel] [lxd/master] Silence rebalance warning not leader

[freeekanayaka on Github]

A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200116/e836c92c/attachment.bin>
-------------- next part --------------
From fd4617adaa58ca9e347cb748071e66cf11f25864 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 16 Jan 2020 16:05:40 +0000
Subject: [PATCH 1/3] Make cluster.Rebalance fail immediately if not leader

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 lxd/cluster/membership.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lxd/cluster/membership.go b/lxd/cluster/membership.go
index 996a1c64d5..ed08b127da 100644
--- a/lxd/cluster/membership.go
+++ b/lxd/cluster/membership.go
@@ -479,11 +479,18 @@ func notifyNodesUpdate(raftNodes []db.RaftNode, id uint64, cert *shared.CertInfo
 // If there's such spare node, return its address as well as the new list of
 // raft nodes.
 func Rebalance(state *state.State, gateway *Gateway) (string, []db.RaftNode, error) {
+	// First get the current raft members, since this method should be
+	// called after a node has left.
+	currentRaftNodes, err := gateway.currentRaftNodes()
+	if err != nil {
+		return "", nil, errors.Wrap(err, "failed to get current raft nodes")
+	}
+
 	// Fetch the nodes from the database, to get their last heartbeat
 	// timestamp and check whether they are offline.
 	nodesByAddress := map[string]db.NodeInfo{}
 	var offlineThreshold time.Duration
-	err := state.Cluster.Transaction(func(tx *db.ClusterTx) error {
+	err = state.Cluster.Transaction(func(tx *db.ClusterTx) error {
 		config, err := ConfigLoad(tx)
 		if err != nil {
 			return errors.Wrap(err, "failed load cluster configuration")
@@ -502,13 +509,6 @@ func Rebalance(state *state.State, gateway *Gateway) (string, []db.RaftNode, err
 		return "", nil, err
 	}
 
-	// First get the current raft members, since this method should be
-	// called after a node has left.
-	currentRaftNodes, err := gateway.currentRaftNodes()
-	if err != nil {
-		return "", nil, errors.Wrap(err, "failed to get current raft nodes")
-	}
-
 	// Group by role. If a node is offline, we'll try to demote it right
 	// away.
 	voters := make([]string, 0)

From e6e584d64acab30174fbb1f72bf84e7b9e30f3fc Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 16 Jan 2020 16:06:19 +0000
Subject: [PATCH 2/3] Export cluster.ErrNotLeader

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 lxd/cluster/gateway.go   | 8 ++++----
 lxd/cluster/heartbeat.go | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lxd/cluster/gateway.go b/lxd/cluster/gateway.go
index 023593247f..194e9da661 100644
--- a/lxd/cluster/gateway.go
+++ b/lxd/cluster/gateway.go
@@ -723,8 +723,8 @@ func (g *Gateway) isLeader() (bool, error) {
 	return leader != nil && leader.ID == g.info.ID, nil
 }
 
-// Internal error signalling that a node not the leader.
-var errNotLeader = fmt.Errorf("Not leader")
+// ErrNotLeader signals that a node not the leader.
+var ErrNotLeader = fmt.Errorf("Not leader")
 
 // Return information about the LXD nodes that a currently part of the raft
 // cluster, as configured in the raft log. It returns an error if this node is
@@ -734,7 +734,7 @@ func (g *Gateway) currentRaftNodes() ([]db.RaftNode, error) {
 	defer g.lock.RUnlock()
 
 	if g.info == nil || g.info.Role != db.RaftVoter {
-		return nil, errNotLeader
+		return nil, ErrNotLeader
 	}
 
 	isLeader, err := g.isLeader()
@@ -742,7 +742,7 @@ func (g *Gateway) currentRaftNodes() ([]db.RaftNode, error) {
 		return nil, err
 	}
 	if !isLeader {
-		return nil, errNotLeader
+		return nil, ErrNotLeader
 	}
 	client, err := g.getClient()
 	if err != nil {
diff --git a/lxd/cluster/heartbeat.go b/lxd/cluster/heartbeat.go
index e00a7e1764..27aeb80767 100644
--- a/lxd/cluster/heartbeat.go
+++ b/lxd/cluster/heartbeat.go
@@ -194,7 +194,7 @@ func (g *Gateway) heartbeat(ctx context.Context, initialHeartbeat bool) {
 	}
 
 	raftNodes, err := g.currentRaftNodes()
-	if err == errNotLeader {
+	if err == ErrNotLeader {
 		return
 	}
 

From 1bebfb93cfab723d28433fd86b3bcbf7710b74f8 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Thu, 16 Jan 2020 16:10:06 +0000
Subject: [PATCH 3/3] Silence warning about failing to rebalance when not
 leader

Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
 lxd/daemon.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lxd/daemon.go b/lxd/daemon.go
index 88b2c1142d..646f1d82df 100644
--- a/lxd/daemon.go
+++ b/lxd/daemon.go
@@ -1482,7 +1482,7 @@ func (d *Daemon) NodeRefreshTask(heartbeatData *cluster.APIHeartbeat) {
 				d.clusterMembershipMutex.Lock()
 				defer d.clusterMembershipMutex.Unlock()
 				err := rebalanceMemberRoles(d)
-				if err != nil {
+				if err != nil && errors.Cause(err) != cluster.ErrNotLeader {
 					logger.Warnf("Could not rebalance cluster member roles: %v", err)
 				}
 			}()