[lxc-devel] [lxd/master] Clustering placement
freeekanayaka on Github
lxc-bot at linuxcontainers.org
Tue Mar 13 08:02:33 UTC 2018
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 427 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20180313/c60b051a/attachment.bin>
-------------- next part --------------
From 7feb6f15cf1f2ce14fd000738653984419e74bda Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Tue, 13 Mar 2018 06:58:26 +0000
Subject: [PATCH 1/2] Add new NodesWithLeastContainers db API
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/db/node.go | 30 ++++++++++++++++++++++++++++++
lxd/db/node_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 74 insertions(+)
diff --git a/lxd/db/node.go b/lxd/db/node.go
index 5898c6fd6..5578090fd 100644
--- a/lxd/db/node.go
+++ b/lxd/db/node.go
@@ -391,6 +391,36 @@ func (c *ClusterTx) NodeOfflineThreshold() (time.Duration, error) {
return threshold, nil
}
+// NodeWithLeastContainers returns the name of the non-offline node with
+// with the least number of containers.
+func (c *ClusterTx) NodeWithLeastContainers() (string, error) {
+ threshold, err := c.NodeOfflineThreshold()
+ if err != nil {
+ return "", errors.Wrap(err, "failed to get offline threshold")
+ }
+ nodes, err := c.Nodes()
+ if err != nil {
+ return "", errors.Wrap(err, "failed to get current nodes")
+ }
+
+ name := ""
+ containers := -1
+ for _, node := range nodes {
+ if node.IsOffline(threshold) {
+ continue
+ }
+ count, err := query.Count(c.tx, "containers", "node_id=?", node.ID)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to get containers count")
+ }
+ if containers == -1 || count < containers {
+ containers = count
+ name = node.Name
+ }
+ }
+ return name, nil
+}
+
func nodeIsOffline(threshold time.Duration, heartbeat time.Time) bool {
return heartbeat.Before(time.Now().Add(-threshold))
}
diff --git a/lxd/db/node_test.go b/lxd/db/node_test.go
index 9ef0fb724..9d5356e7d 100644
--- a/lxd/db/node_test.go
+++ b/lxd/db/node_test.go
@@ -216,3 +216,47 @@ INSERT INTO images_nodes(image_id, node_id) VALUES(1, 1)`)
require.NoError(t, err)
assert.Equal(t, "", message)
}
+
+// If there are 2 online nodes, return the address of the one with the least
+// number of containers.
+func TestNodeWithLeastContainers(t *testing.T) {
+ tx, cleanup := db.NewTestClusterTx(t)
+ defer cleanup()
+
+ _, err := tx.NodeAdd("buzz", "1.2.3.4:666")
+ require.NoError(t, err)
+
+ // Add a container to the default node (ID 1)
+ _, err = tx.Tx().Exec(`
+INSERT INTO containers (id, node_id, name, architecture, type) VALUES (1, 1, 'foo', 1, 1)
+`)
+ require.NoError(t, err)
+
+ name, err := tx.NodeWithLeastContainers()
+ require.NoError(t, err)
+ assert.Equal(t, "buzz", name)
+}
+
+// If there are nodes, and one of them is offline, return the name of the
+// online node, even if the offline one has more containers.
+func TestNodeWithLeastContainers_OfflineNode(t *testing.T) {
+ tx, cleanup := db.NewTestClusterTx(t)
+ defer cleanup()
+
+ id, err := tx.NodeAdd("buzz", "1.2.3.4:666")
+ require.NoError(t, err)
+
+ // Add a container to the newly created node.
+ _, err = tx.Tx().Exec(`
+INSERT INTO containers (id, node_id, name, architecture, type) VALUES (1, ?, 'foo', 1, 1)
+`, id)
+ require.NoError(t, err)
+
+ // Mark the default node has offline.
+ err = tx.NodeHeartbeat("0.0.0.0", time.Now().Add(-time.Minute))
+ require.NoError(t, err)
+
+ name, err := tx.NodeWithLeastContainers()
+ require.NoError(t, err)
+ assert.Equal(t, "buzz", name)
+}
From 3cdb257f5edbf3a6af8f5e2989819d31954aaab2 Mon Sep 17 00:00:00 2001
From: Free Ekanayaka <free.ekanayaka at canonical.com>
Date: Tue, 13 Mar 2018 07:39:24 +0000
Subject: [PATCH 2/2] Select the node with the least number of containers when
no target is given
Signed-off-by: Free Ekanayaka <free.ekanayaka at canonical.com>
---
lxd/containers_post.go | 17 ++++++++++++++++-
test/suites/clustering.sh | 14 ++++++++++++++
2 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/lxd/containers_post.go b/lxd/containers_post.go
index 3f9ebc10b..c31aba8de 100644
--- a/lxd/containers_post.go
+++ b/lxd/containers_post.go
@@ -525,6 +525,21 @@ func containersPost(d *Daemon, r *http.Request) Response {
}
targetNode := r.FormValue("target")
+ if targetNode == "" {
+ // If no target node was specified, pick the node with the
+ // least number of containers. If there's just one node, or if
+ // the selected node is the local one, this is effectively a
+ // no-op, since NodeWithLeastContainers() will return an empty
+ // string.
+ err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
+ var err error
+ targetNode, err = tx.NodeWithLeastContainers()
+ return err
+ })
+ if err != nil {
+ return SmartError(err)
+ }
+ }
if targetNode != "" {
address, err := cluster.ResolveTarget(d.cluster, targetNode)
if err != nil {
@@ -537,7 +552,7 @@ func containersPost(d *Daemon, r *http.Request) Response {
return SmartError(err)
}
logger.Debugf("Forward container post request to %s", address)
- op, err := client.CreateContainer(req)
+ op, err := client.UseTarget(targetNode).CreateContainer(req)
if err != nil {
return SmartError(err)
}
diff --git a/test/suites/clustering.sh b/test/suites/clustering.sh
index b0122f3e5..cd16ecd52 100644
--- a/test/suites/clustering.sh
+++ b/test/suites/clustering.sh
@@ -245,6 +245,20 @@ test_clustering_containers() {
LXD_DIR="${LXD_ONE_DIR}" lxc list | grep foo | grep -q ERROR
LXD_DIR="${LXD_ONE_DIR}" lxc config set cluster.offline_threshold 20
+ # Start a container without specifying any target. It will be placed
+ # on node1 since node2 is offline and both node1 and node3 have zero
+ # containers, but node1 has a lower node ID.
+ LXD_DIR="${LXD_THREE_DIR}" lxc launch testimage bar
+ LXD_DIR="${LXD_THREE_DIR}" lxc info bar | grep -q "Location: node1"
+
+ # Start a container without specifying any target. It will be placed
+ # on node3 since node2 is offline and node1 already has a container.
+ LXD_DIR="${LXD_THREE_DIR}" lxc launch testimage egg
+ LXD_DIR="${LXD_THREE_DIR}" lxc info egg | grep -q "Location: node3"
+
+ LXD_DIR="${LXD_ONE_DIR}" lxc stop egg --force
+ LXD_DIR="${LXD_ONE_DIR}" lxc stop bar --force
+
LXD_DIR="${LXD_THREE_DIR}" lxd shutdown
LXD_DIR="${LXD_ONE_DIR}" lxd shutdown
sleep 2
More information about the lxc-devel
mailing list