[lxc-devel] [lxd/master] Extend cluster
ralubis on Github
lxc-bot at linuxcontainers.org
Wed Nov 20 22:10:37 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 322 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20191120/c12dbfd1/attachment-0001.bin>
-------------- next part --------------
From fe8327b767fdb391f082ae1883e2e5f6d9f11581 Mon Sep 17 00:00:00 2001
From: Rizwan Ahmad Lubis <rizwan.lubis at gmail.com>
Date: Sun, 3 Nov 2019 00:29:02 -0500
Subject: [PATCH 1/2] Issue #6230
---
lxd/api_cluster.go | 26 ++++++++++++++++++++++++--
lxd/cluster/membership.go | 32 ++++++++++++++++++++++++++------
lxd/main_init_interactive.go | 2 +-
3 files changed, 51 insertions(+), 9 deletions(-)
diff --git a/lxd/api_cluster.go b/lxd/api_cluster.go
index ea2c2d3aa1..96ab18d5fb 100644
--- a/lxd/api_cluster.go
+++ b/lxd/api_cluster.go
@@ -608,6 +608,12 @@ func clusterPutJoin(d *Daemon, req api.ClusterPut) response.Response {
// Add the cluster flag from the agent
version.UserAgentFeatures([]string{"cluster"})
+ logger.Infof("Making cluster rebalance call.")
+ err = clusterRebalance(client)
+ if err != nil {
+ logger.Errorf("Failed cluster rebalance call: %v", err)
+ }
+
return nil
}
@@ -820,6 +826,14 @@ func clusterAcceptMember(
return info, nil
}
+func clusterRebalance(client lxd.InstanceServer) error {
+ _, _, err := client.RawQuery("POST", "/internal/cluster/rebalance", nil, "")
+ if err != nil {
+ return errors.Wrap(err, "Failed cluster rebalance request")
+ }
+ return nil
+}
+
func clusterNodesGet(d *Daemon, r *http.Request) response.Response {
recursion := util.IsRecursionRequest(r)
@@ -1075,7 +1089,7 @@ func internalClusterPostRebalance(d *Daemon, r *http.Request) response.Response
return response.InternalError(err)
}
if localAddress != leader {
- logger.Debugf("Redirect cluster rebalance request to %s", leader)
+ logger.Infof("Redirect cluster rebalance request to %s", leader)
url := &url.URL{
Scheme: "https",
Path: "/internal/cluster/rebalance",
@@ -1084,7 +1098,7 @@ func internalClusterPostRebalance(d *Daemon, r *http.Request) response.Response
return response.SyncResponseRedirect(url.String())
}
- logger.Debugf("Rebalance cluster")
+ logger.Infof("Handling rebalance cluster request")
// Check if we have a spare node to promote.
address, nodes, err := cluster.Rebalance(d.State(), d.gateway)
@@ -1094,6 +1108,7 @@ func internalClusterPostRebalance(d *Daemon, r *http.Request) response.Response
if address == "" {
// If no node could be found to promote, notify all nodes about current set of DB nodes
+ logger.Infof("No address found to promote")
var offlineThreshold time.Duration
err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
var err error
@@ -1128,6 +1143,8 @@ func internalClusterPostRebalance(d *Daemon, r *http.Request) response.Response
return response.SyncResponse(true, nil)
}
+ logger.Infof("Promoting address %s. New raft node list: %v", address, nodes)
+
// Tell the node to promote itself.
post := &internalClusterPostPromoteRequest{}
for _, node := range nodes {
@@ -1144,14 +1161,18 @@ func internalClusterPostRebalance(d *Daemon, r *http.Request) response.Response
}
_, _, err = client.RawQuery("POST", "/internal/cluster/promote", post, "")
if err != nil {
+ logger.Errorf("Promote request failed: %v", err)
return response.SmartError(err)
}
+ logger.Infof("Promote request succeeded")
return response.SyncResponse(true, nil)
}
// Used to promote the local non-database node to be a database one.
func internalClusterPostPromote(d *Daemon, r *http.Request) response.Response {
+ logger.Infof("Handling promote request")
+
req := internalClusterPostPromoteRequest{}
// Parse the request
@@ -1162,6 +1183,7 @@ func internalClusterPostPromote(d *Daemon, r *http.Request) response.Response {
// Sanity checks
if len(req.RaftNodes) == 0 {
+ logger.Errorf("No raft nodes provided")
return response.BadRequest(fmt.Errorf("No raft members provided"))
}
diff --git a/lxd/cluster/membership.go b/lxd/cluster/membership.go
index b8b4456c1f..d5865650d0 100644
--- a/lxd/cluster/membership.go
+++ b/lxd/cluster/membership.go
@@ -82,6 +82,8 @@ func Bootstrap(state *state.State, gateway *Gateway, name string) error {
}
// Update our role list.
+ fmt.Println("Testing that I am running from edited source.\n")
+
err = tx.NodeAddRole(1, db.ClusterRoleDatabase)
if err != nil {
return errors.Wrapf(err, "Failed to add database role for the node")
@@ -198,8 +200,10 @@ func Accept(state *state.State, gateway *Gateway, name, address string, schema,
if err != nil {
return nil, errors.Wrap(err, "Failed to get raft nodes from the log")
}
-
- if len(nodes) < membershipMaxRaftNodes {
+ count, err := Count(state)
+ if count != 2 && len(nodes) < membershipMaxRaftNodes {
+// return nil, fmt.Errorf("%v", nodes)
+// if false {
err = state.Node.Transaction(func(tx *db.NodeTx) error {
id, err := tx.RaftNodeAdd(address)
if err != nil {
@@ -482,8 +486,9 @@ func Rebalance(state *state.State, gateway *Gateway) (string, []db.RaftNode, err
if err != nil {
return "", nil, errors.Wrap(err, "failed to get current raft nodes")
}
- if len(currentRaftNodes) >= membershipMaxRaftNodes {
- // We're already at full capacity.
+ logger.Infof("List of current raft nodes: %v", currentRaftNodes)
+ if len(currentRaftNodes) >= membershipMaxRaftNodes || len(currentRaftNodes) == 1 {
+ // We're already at full capacity or would have a two-member cluster.
return "", nil, nil
}
@@ -506,12 +511,14 @@ func Rebalance(state *state.State, gateway *Gateway) (string, []db.RaftNode, err
// Find a node that is not part of the raft cluster yet.
for _, node := range nodes {
if shared.StringInSlice(node.Address, currentRaftAddresses) {
+ logger.Infof("node %s (%s) is already a database node", node.Name, node.Address)
continue // This is already a database node
}
if node.IsOffline(config.OfflineThreshold()) {
+ logger.Infof("node %s (%s) is offline", node.Name, node.Address)
continue // This node is offline
}
- logger.Debugf(
+ logger.Infof(
"Found spare node %s (%s) to be promoted as database node", node.Name, node.Address)
address = node.Address
break
@@ -531,6 +538,7 @@ func Rebalance(state *state.State, gateway *Gateway) (string, []db.RaftNode, err
// Figure out the next ID in the raft_nodes table
var updatedRaftNodes []db.RaftNode
err = gateway.db.Transaction(func(tx *db.NodeTx) error {
+ logger.Infof("Calling raft node add.")
id, err := tx.RaftNodeAdd(address)
if err != nil {
return errors.Wrap(err, "Failed to add new raft node")
@@ -549,7 +557,7 @@ func Rebalance(state *state.State, gateway *Gateway) (string, []db.RaftNode, err
// Promote makes a LXD node which is not a database node, become part of the
// raft cluster.
func Promote(state *state.State, gateway *Gateway, nodes []db.RaftNode) error {
- logger.Info("Promote node to database node")
+ logger.Info("Promote this node to database node")
// Sanity check that this is not already a database node
if gateway.IsDatabaseNode() {
@@ -597,6 +605,7 @@ func Promote(state *state.State, gateway *Gateway, nodes []db.RaftNode) error {
// includes ourselves). This will make the gateway start a raft node
// when restarted.
err = state.Node.Transaction(func(tx *db.NodeTx) error {
+ logger.Infof("Replacing raft node list")
err = tx.RaftNodesReplace(nodes)
if err != nil {
return errors.Wrap(err, "failed to set raft nodes")
@@ -642,6 +651,7 @@ func Promote(state *state.State, gateway *Gateway, nodes []db.RaftNode) error {
return errors.Wrap(err, "Failed to connect to cluster leader")
}
defer client.Close()
+ logger.Info("Found leader successfully. Adding gateway info.")
err = client.Add(ctx, gateway.raft.info)
if err != nil {
@@ -650,8 +660,18 @@ func Promote(state *state.State, gateway *Gateway, nodes []db.RaftNode) error {
// Unlock regular access to our cluster database and add the database role.
err = state.Cluster.ExitExclusive(func(tx *db.ClusterTx) error {
+ logger.Info("Getting node list.")
+ nodeInfo, err := tx.Nodes()
+ if err != nil {
+ logger.Errorf("Failed to get node list: %v", err)
+ return errors.Wrapf(err, "Failed to get node list")
+ }
+ logger.Info("Test %d. Got node list: %+v", 0, nodeInfo)
+
+ logger.Info("Adding database role to this node")
err = tx.NodeAddRole(id, db.ClusterRoleDatabase)
if err != nil {
+ logger.Errorf("Failed to add database role: %v", err)
return errors.Wrapf(err, "Failed to add database role for the node")
}
return err
diff --git a/lxd/main_init_interactive.go b/lxd/main_init_interactive.go
index f3947ed526..b998b18c82 100644
--- a/lxd/main_init_interactive.go
+++ b/lxd/main_init_interactive.go
@@ -99,7 +99,7 @@ func (c *cmdInit) RunInteractive(cmd *cobra.Command, args []string, d lxd.Instan
}
func (c *cmdInit) askClustering(config *cmdInitData, d lxd.InstanceServer) error {
- if cli.AskBool("Would you like to use LXD clustering? (yes/no) [default=no]: ", "no") {
+ if cli.AskBool("Would you like to use LXD clustering? TEST (yes/no) [default=no]: ", "no") {
config.Cluster = &initDataCluster{}
config.Cluster.Enabled = true
From 7e94399a6c668ae6104e76210c61417794230c0a Mon Sep 17 00:00:00 2001
From: Rizwan Ahmad Lubis <rizwan.lubis at gmail.com>
Date: Wed, 20 Nov 2019 14:51:32 -0600
Subject: [PATCH 2/2] clean up
---
lxd/cluster/membership.go | 4 ----
lxd/main_init_interactive.go | 2 +-
2 files changed, 1 insertion(+), 5 deletions(-)
diff --git a/lxd/cluster/membership.go b/lxd/cluster/membership.go
index d5865650d0..beaee295e4 100644
--- a/lxd/cluster/membership.go
+++ b/lxd/cluster/membership.go
@@ -82,8 +82,6 @@ func Bootstrap(state *state.State, gateway *Gateway, name string) error {
}
// Update our role list.
- fmt.Println("Testing that I am running from edited source.\n")
-
err = tx.NodeAddRole(1, db.ClusterRoleDatabase)
if err != nil {
return errors.Wrapf(err, "Failed to add database role for the node")
@@ -202,8 +200,6 @@ func Accept(state *state.State, gateway *Gateway, name, address string, schema,
}
count, err := Count(state)
if count != 2 && len(nodes) < membershipMaxRaftNodes {
-// return nil, fmt.Errorf("%v", nodes)
-// if false {
err = state.Node.Transaction(func(tx *db.NodeTx) error {
id, err := tx.RaftNodeAdd(address)
if err != nil {
diff --git a/lxd/main_init_interactive.go b/lxd/main_init_interactive.go
index b998b18c82..f3947ed526 100644
--- a/lxd/main_init_interactive.go
+++ b/lxd/main_init_interactive.go
@@ -99,7 +99,7 @@ func (c *cmdInit) RunInteractive(cmd *cobra.Command, args []string, d lxd.Instan
}
func (c *cmdInit) askClustering(config *cmdInitData, d lxd.InstanceServer) error {
- if cli.AskBool("Would you like to use LXD clustering? TEST (yes/no) [default=no]: ", "no") {
+ if cli.AskBool("Would you like to use LXD clustering? (yes/no) [default=no]: ", "no") {
config.Cluster = &initDataCluster{}
config.Cluster.Enabled = true
More information about the lxc-devel
mailing list