[lxc-devel] [lxd/master] Small CEPH improvements
stgraber on Github
lxc-bot at linuxcontainers.org
Thu Jul 18 00:12:25 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20190717/b3620cb2/attachment-0001.bin>
-------------- next part --------------
From 00daff533635a7dd908977d3c7d1eaffe991cf60 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jul 2019 18:23:46 -0400
Subject: [PATCH 1/2] lxd/storage/ceph: Handle EBUSY on unmap
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/storage_ceph_utils.go | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/lxd/storage_ceph_utils.go b/lxd/storage_ceph_utils.go
index 2ae569d358..9c458ad929 100644
--- a/lxd/storage_ceph_utils.go
+++ b/lxd/storage_ceph_utils.go
@@ -9,6 +9,7 @@ import (
"strconv"
"strings"
"syscall"
+ "time"
"github.com/pborman/uuid"
"golang.org/x/sys/unix"
@@ -169,6 +170,8 @@ func cephRBDVolumeUnmap(clusterName string, poolName string, volumeName string,
volumeType string, userName string, unmapUntilEINVAL bool) error {
unmapImageName := fmt.Sprintf("%s_%s", volumeType, volumeName)
+ busyCount := 0
+
again:
_, err := shared.RunCommand(
"rbd",
@@ -187,8 +190,21 @@ again:
// EINVAL (already unmapped)
return nil
}
+
+ if waitStatus.ExitStatus() == 16 {
+ // EBUSY (currently in use)
+ busyCount++
+ if busyCount == 10 {
+ return err
+ }
+
+ // Wait a second an try again
+ time.Sleep(time.Second)
+ goto again
+ }
}
}
+
return err
}
From 36b75fba9acd6e62de5bc905ec57649f1611e64f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Wed, 17 Jul 2019 19:18:20 -0400
Subject: [PATCH 2/2] lxd/storage/ceph: Slightly speed up creation
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/storage_ceph.go | 65 ++++++++++++++++-----------------------------
1 file changed, 23 insertions(+), 42 deletions(-)
diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go
index 6a1253f102..279ea043a8 100644
--- a/lxd/storage_ceph.go
+++ b/lxd/storage_ceph.go
@@ -136,11 +136,8 @@ func (s *storageCeph) StoragePoolCreate() error {
if !cephOSDPoolExists(s.ClusterName, s.OSDPoolName, s.UserName) {
logger.Debugf(`CEPH OSD storage pool "%s" does not exist`, s.OSDPoolName)
- // create new osd pool
- msg, err := shared.TryRunCommand("ceph", "--name",
- fmt.Sprintf("client.%s", s.UserName), "--cluster",
- s.ClusterName, "osd", "pool", "create", s.OSDPoolName,
- s.PGNum)
+ // Create new osd pool
+ msg, err := shared.TryRunCommand("ceph", "--name", fmt.Sprintf("client.%s", s.UserName), "--cluster", s.ClusterName, "osd", "pool", "create", s.OSDPoolName, s.PGNum)
if err != nil {
logger.Errorf(`Failed to create CEPH osd storage pool "%s" in cluster "%s": %s`, s.OSDPoolName, s.ClusterName, msg)
return err
@@ -152,27 +149,39 @@ func (s *storageCeph) StoragePoolCreate() error {
return
}
- err := cephOSDPoolDestroy(s.ClusterName, s.OSDPoolName,
- s.UserName)
+ err := cephOSDPoolDestroy(s.ClusterName, s.OSDPoolName, s.UserName)
if err != nil {
logger.Warnf(`Failed to delete ceph storage pool "%s" in cluster "%s": %s`, s.OSDPoolName, s.ClusterName, err)
}
}()
+ // Create dummy storage volume. Other LXD instances will use this to detect whether this osd pool is already in use by another LXD instance.
+ err = cephRBDVolumeCreate(s.ClusterName, s.OSDPoolName, s.OSDPoolName, "lxd", "0", s.UserName)
+ if err != nil {
+ logger.Errorf(`Failed to create RBD storage volume "%s" on storage pool "%s": %s`, s.pool.Name, s.pool.Name, err)
+ return err
+ }
+ s.pool.Config["volatile.pool.pristine"] = "true"
+ logger.Debugf(`Created RBD storage volume "%s" on storage pool "%s"`, s.pool.Name, s.pool.Name)
} else {
logger.Debugf(`CEPH OSD storage pool "%s" does exist`, s.OSDPoolName)
- // use existing osd pool
- msg, err := shared.RunCommand("ceph", "--name",
- fmt.Sprintf("client.%s", s.UserName),
- "--cluster", s.ClusterName, "osd", "pool", "get",
- s.OSDPoolName, "pg_num")
+ ok := cephRBDVolumeExists(s.ClusterName, s.OSDPoolName, s.OSDPoolName, "lxd", s.UserName)
+ s.pool.Config["volatile.pool.pristine"] = "false"
+ if ok {
+ if s.pool.Config["ceph.osd.force_reuse"] == "" || !shared.IsTrue(s.pool.Config["ceph.osd.force_reuse"]) {
+ return fmt.Errorf("CEPH OSD storage pool \"%s\" in cluster \"%s\" seems to be in use by another LXD instance. Use \"ceph.osd.force_reuse=true\" to force.", s.pool.Name, s.ClusterName)
+ }
+ }
+
+ // Use existing osd pool
+ msg, err := shared.RunCommand("ceph", "--name", fmt.Sprintf("client.%s", s.UserName), "--cluster", s.ClusterName, "osd", "pool", "get", s.OSDPoolName, "pg_num")
if err != nil {
logger.Errorf(`Failed to retrieve number of placement groups for CEPH osd storage pool "%s" in cluster "%s": %s`, s.OSDPoolName, s.ClusterName, msg)
return err
}
- logger.Debugf(`Retrieved number of placement groups or CEPH osd storage pool "%s" in cluster "%s"`, s.OSDPoolName, s.ClusterName)
+ logger.Debugf(`Retrieved number of placement groups or CEPH osd storage pool "%s" in cluster "%s"`, s.OSDPoolName, s.ClusterName)
idx := strings.Index(msg, "pg_num:")
if idx == -1 {
logger.Errorf(`Failed to parse number of placement groups for CEPH osd storage pool "%s" in cluster "%s": %s`, s.OSDPoolName, s.ClusterName, msg)
@@ -180,6 +189,7 @@ func (s *storageCeph) StoragePoolCreate() error {
msg = msg[(idx + len("pg_num:")):]
msg = strings.TrimSpace(msg)
+
// It is ok to update the pool configuration since storage pool
// creation via API is implemented such that the storage pool is
// checked for a changed config after this function returns and
@@ -225,35 +235,6 @@ func (s *storageCeph) StoragePoolCreate() error {
}
}()
- ok := cephRBDVolumeExists(s.ClusterName, s.OSDPoolName, s.OSDPoolName,
- "lxd", s.UserName)
- s.pool.Config["volatile.pool.pristine"] = "false"
- if !ok {
- s.pool.Config["volatile.pool.pristine"] = "true"
- // Create dummy storage volume. Other LXD instances will use
- // this to detect whether this osd pool is already in use by
- // another LXD instance.
- err = cephRBDVolumeCreate(s.ClusterName, s.OSDPoolName,
- s.OSDPoolName, "lxd", "0", s.UserName)
- if err != nil {
- logger.Errorf(`Failed to create RBD storage volume "%s" on storage pool "%s": %s`, s.pool.Name, s.pool.Name, err)
- return err
- }
- logger.Debugf(`Created RBD storage volume "%s" on storage pool "%s"`, s.pool.Name, s.pool.Name)
- } else {
- msg := fmt.Sprintf(`CEPH OSD storage pool "%s" in cluster `+
- `"%s" seems to be in use by another LXD instace`,
- s.pool.Name, s.ClusterName)
- if s.pool.Config["ceph.osd.force_reuse"] == "" ||
- !shared.IsTrue(s.pool.Config["ceph.osd.force_reuse"]) {
- msg += `. Set "ceph.osd.force_reuse=true" to force ` +
- `LXD to reuse the pool`
- logger.Errorf(msg)
- return fmt.Errorf(msg)
- }
- logger.Warnf(msg)
- }
-
logger.Infof(`Created CEPH OSD storage pool "%s" in cluster "%s"`,
s.pool.Name, s.ClusterName)
More information about the lxc-devel
mailing list