[lxc-devel] [lxd/master] Drop database role from nodes roles table
freeekanayaka on Github
lxc-bot at linuxcontainers.org
Tue Jun 30 06:49:41 UTC 2020
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20200629/1d59cf2c/attachment.bin>
-------------- next part --------------
From 6ca22a1973597afb2e5b27212489c90c69d7b7d4 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Tue, 30 Jun 2020 08:47:13 +0200
Subject: [PATCH 1/2] lxd/db: Drop ClusterRoleDatabase records from the
database
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/db/cluster/schema.go | 2 +-
lxd/db/cluster/update.go | 11 +++++++++++
lxd/db/node.go | 8 +++++---
3 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/lxd/db/cluster/schema.go b/lxd/db/cluster/schema.go
index 201ed4e1eb..1d6e58c98d 100644
--- a/lxd/db/cluster/schema.go
+++ b/lxd/db/cluster/schema.go
@@ -565,5 +565,5 @@ CREATE TABLE storage_volumes_snapshots_config (
UNIQUE (storage_volume_snapshot_id, key)
);
-INSERT INTO schema (version, updated_at) VALUES (31, strftime("%s"))
+INSERT INTO schema (version, updated_at) VALUES (32, strftime("%s"))
`
diff --git a/lxd/db/cluster/update.go b/lxd/db/cluster/update.go
index b6d5d4b1fe..c1ce123a16 100644
--- a/lxd/db/cluster/update.go
+++ b/lxd/db/cluster/update.go
@@ -68,6 +68,17 @@ var updates = map[int]schema.Update{
29: updateFromV28,
30: updateFromV29,
31: updateFromV30,
+ 32: updateFromV31,
+}
+
+// Drop database role from the nodes_roles table, since we now rely on the raft
+// log instead.
+func updateFromV31(tx *sql.Tx) error {
+ _, err := tx.Exec("DELETE FROM nodes_roles WHERE role = 0")
+ if err != nil {
+ return err
+ }
+ return nil
}
// Add content type field to storage volumes
diff --git a/lxd/db/node.go b/lxd/db/node.go
index 1e22920877..e0a2c30f41 100644
--- a/lxd/db/node.go
+++ b/lxd/db/node.go
@@ -25,9 +25,11 @@ type ClusterRole string
const ClusterRoleDatabase = ClusterRole("database")
// ClusterRoles maps role ids into human-readable names.
-var ClusterRoles = map[int]ClusterRole{
- 0: ClusterRoleDatabase,
-}
+//
+// Note: the database role is currently stored directly in the raft
+// configuration which acts as single source of truth for it. This map should
+// only contain LXD-specific cluster roles.
+var ClusterRoles = map[int]ClusterRole{}
// NodeInfo holds information about a single LXD instance in a cluster.
type NodeInfo struct {
From 60347fb5999eb0368c2277a1efd55e4f8120fe16 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Tue, 30 Jun 2020 08:48:15 +0200
Subject: [PATCH 2/2] lxd/cluster: Fetch database role information directly
from raft
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/api_cluster.go | 4 +-
lxd/cluster/membership.go | 72 +++++++++++++++++-----------------
lxd/cluster/membership_test.go | 2 +-
lxd/patches.go | 48 -----------------------
4 files changed, 39 insertions(+), 87 deletions(-)
diff --git a/lxd/api_cluster.go b/lxd/api_cluster.go
index 527da84e7b..fbc61dd979 100644
--- a/lxd/api_cluster.go
+++ b/lxd/api_cluster.go
@@ -840,7 +840,7 @@ func clusterAcceptMember(
func clusterNodesGet(d *Daemon, r *http.Request) response.Response {
recursion := util.IsRecursionRequest(r)
- nodes, err := cluster.List(d.State())
+ nodes, err := cluster.List(d.State(), d.gateway)
if err != nil {
return response.SmartError(err)
}
@@ -863,7 +863,7 @@ func clusterNodesGet(d *Daemon, r *http.Request) response.Response {
func clusterNodeGet(d *Daemon, r *http.Request) response.Response {
name := mux.Vars(r)["name"]
- nodes, err := cluster.List(d.State())
+ nodes, err := cluster.List(d.State(), d.gateway)
if err != nil {
return response.SmartError(err)
}
diff --git a/lxd/cluster/membership.go b/lxd/cluster/membership.go
index dff1e3f0ef..1d260d4295 100644
--- a/lxd/cluster/membership.go
+++ b/lxd/cluster/membership.go
@@ -81,12 +81,6 @@ func Bootstrap(state *state.State, gateway *Gateway, name string) error {
return errors.Wrap(err, "failed to update cluster node")
}
- // Update our role list.
- err = tx.CreateNodeRole(1, db.ClusterRoleDatabase)
- if err != nil {
- return errors.Wrapf(err, "Failed to add database role for the node")
- }
-
return nil
})
if err != nil {
@@ -446,14 +440,6 @@ func Join(state *state.State, gateway *Gateway, cert *shared.CertInfo, name stri
return errors.Wrapf(err, "failed to unmark the node as pending")
}
- // Update our role list if needed.
- if info.Role == db.RaftVoter {
- err = tx.CreateNodeRole(node.ID, db.ClusterRoleDatabase)
- if err != nil {
- return errors.Wrapf(err, "Failed to add database role for the node")
- }
- }
-
// Generate partial heartbeat request containing just a raft node list.
notifyNodesUpdate(raftNodes, info.ID, cert)
@@ -553,14 +539,6 @@ func Rebalance(state *state.State, gateway *Gateway) (string, []db.RaftNode, err
if err != nil {
return "", nil, errors.Wrap(err, "Failed to demote offline node")
}
- if info.Role == db.RaftVoter {
- err := state.Cluster.Transaction(func(tx *db.ClusterTx) error {
- return tx.RemoveNodeRole(node.ID, db.ClusterRoleDatabase)
- })
- if err != nil {
- return "", nil, errors.Wrap(err, "Failed to update node role")
- }
- }
currentRaftNodes[i].Role = db.RaftSpare
}
continue
@@ -762,19 +740,9 @@ assign:
gateway.info = info
- // Unlock regular access to our cluster database and add the database role.
+ // Unlock regular access to our cluster database.
err = transactor(func(tx *db.ClusterTx) error {
- var f func(id int64, role db.ClusterRole) error
- if info.Role == db.RaftVoter {
- f = tx.CreateNodeRole
- } else {
- f = tx.RemoveNodeRole
- }
- err = f(state.Cluster.GetNodeID(), db.ClusterRoleDatabase)
- if err != nil {
- return errors.Wrapf(err, "Failed to change role for the node")
- }
- return err
+ return nil
})
if err != nil {
return errors.Wrap(err, "Cluster database initialization failed")
@@ -930,7 +898,7 @@ func Purge(cluster *db.Cluster, name string) error {
}
// List the nodes of the cluster.
-func List(state *state.State) ([]api.ClusterMember, error) {
+func List(state *state.State, gateway *Gateway) ([]api.ClusterMember, error) {
var err error
var nodes []db.NodeInfo
var offlineThreshold time.Duration
@@ -952,14 +920,46 @@ func List(state *state.State) ([]api.ClusterMember, error) {
return nil, err
}
+ store := gateway.NodeStore()
+ dial := gateway.DialFunc()
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ cli, err := client.FindLeader(ctx, store, client.WithDialFunc(dial))
+ if err != nil {
+ return nil, err
+ }
+ defer cli.Close()
+
+ raftNodes, err := cli.Cluster(ctx)
+ if err != nil {
+ return nil, err
+ }
+ raftRoles := map[string]client.NodeRole{} // Address to role
+ for _, node := range raftNodes {
+ address := node.Address
+ if address == "1" {
+ addr, err := gateway.raftAddress(1)
+ if err != nil {
+ return nil, err
+ }
+ address = string(addr)
+ }
+ raftRoles[address] = node.Role
+ }
+
result := make([]api.ClusterMember, len(nodes))
now := time.Now()
version := nodes[0].Version()
for i, node := range nodes {
result[i].ServerName = node.Name
result[i].URL = fmt.Sprintf("https://%s", node.Address)
- result[i].Database = shared.StringInSlice(string(db.ClusterRoleDatabase), node.Roles)
+ result[i].Database = raftRoles[node.Address] == db.RaftVoter
result[i].Roles = node.Roles
+ if result[i].Database {
+ result[i].Roles = append(result[i].Roles, string(db.ClusterRoleDatabase))
+ }
result[i].Architecture, err = osarch.ArchitectureName(node.Architecture)
if err != nil {
return nil, err
diff --git a/lxd/cluster/membership_test.go b/lxd/cluster/membership_test.go
index 63826b5f14..8013edba01 100644
--- a/lxd/cluster/membership_test.go
+++ b/lxd/cluster/membership_test.go
@@ -327,7 +327,7 @@ func TestJoin(t *testing.T) {
assert.Equal(t, db.RaftStandBy, raftNodes[1].Role)
// The List function returns all nodes in the cluster.
- nodes, err := cluster.List(state)
+ nodes, err := cluster.List(state, gateway)
require.NoError(t, err)
assert.Len(t, nodes, 2)
assert.Equal(t, "Online", nodes[0].Status)
diff --git a/lxd/patches.go b/lxd/patches.go
index 87ff3d8262..2cbf548307 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -3424,54 +3424,6 @@ func patchStorageApiUpdateContainerSnapshots(name string, d *Daemon) error {
}
func patchClusteringAddRoles(name string, d *Daemon) error {
- addresses := []string{}
- err := d.State().Node.Transaction(func(tx *db.NodeTx) error {
- nodes, err := tx.GetRaftNodes()
- if err != nil {
- return errors.Wrap(err, "Failed to fetch current raft nodes")
- }
-
- for _, node := range nodes {
- addresses = append(addresses, node.Address)
- }
-
- return nil
- })
- if err != nil {
- return err
- }
-
- var nodes []db.NodeInfo
- err = d.State().Cluster.Transaction(func(tx *db.ClusterTx) error {
- nodes, err = tx.GetNodes()
- if err != nil {
- return err
- }
-
- for _, node := range nodes {
- if node.Address == "0.0.0.0" {
- continue
- }
-
- if shared.StringInSlice(node.Address, addresses) && !shared.StringInSlice(string(db.ClusterRoleDatabase), node.Roles) {
- err = tx.CreateNodeRole(node.ID, db.ClusterRoleDatabase)
- if err != nil {
- return err
- }
- } else if shared.StringInSlice(string(db.ClusterRoleDatabase), node.Roles) {
- err = tx.RemoveNodeRole(node.ID, db.ClusterRoleDatabase)
- if err != nil {
- return err
- }
- }
- }
-
- return nil
- })
- if err != nil {
- return err
- }
-
return nil
}
More information about the lxc-devel
mailing list