[lxc-devel] [lxd/master] ceph: use "/dev/rbd<idx" via sysfs
brauner on Github
lxc-bot at linuxcontainers.org
Mon Jul 31 19:09:11 UTC 2017
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 364 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20170731/a1b5d83d/attachment.bin>
-------------- next part --------------
From d854abfb089537d1bbd6bfd890a6fd84c812eac8 Mon Sep 17 00:00:00 2001
From: Christian Brauner <christian.brauner at ubuntu.com>
Date: Mon, 31 Jul 2017 21:08:04 +0200
Subject: [PATCH] ceph: use "/dev/rbd<idx" via sysfs
Signed-off-by: Christian Brauner <christian.brauner at ubuntu.com>
---
lxd/storage_ceph.go | 75 ++++++++++++---------------
lxd/storage_ceph_utils.go | 126 +++++++++++++++++++++++++++++++++++-----------
2 files changed, 128 insertions(+), 73 deletions(-)
diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go
index b721cdca9..bee2775e6 100644
--- a/lxd/storage_ceph.go
+++ b/lxd/storage_ceph.go
@@ -371,8 +371,8 @@ func (s *storageCeph) StoragePoolVolumeCreate() error {
}
}()
- err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, s.volume.Name,
- storagePoolVolumeTypeNameCustom, s.UserName)
+ RBDDevPath, err := cephRBDVolumeMap(s.ClusterName, s.OSDPoolName,
+ s.volume.Name, storagePoolVolumeTypeNameCustom, s.UserName)
if err != nil {
logger.Errorf(`Failed to map RBD storage volume for "%s" on `+
`storage pool "%s": %s`, s.volume.Name, s.pool.Name, err)
@@ -401,14 +401,6 @@ func (s *storageCeph) StoragePoolVolumeCreate() error {
`"%s" on storage pool "%s"`, RBDFilesystem, s.volume.Name,
s.pool.Name)
- // get rbd device path
- RBDDevPath := getRBDDevPath(
- s.OSDPoolName,
- storagePoolVolumeTypeNameCustom,
- s.volume.Name)
- logger.Debugf(`Retrieved device path "%s" of RBD storage volume "%s" `+
- `on storage pool "%s"`, RBDDevPath, s.volume.Name, s.pool.Name)
-
msg, err := makeFSType(RBDDevPath, RBDFilesystem)
if err != nil {
logger.Errorf(`Failed to create filesystem type "%s" on `+
@@ -526,10 +518,6 @@ func (s *storageCeph) StoragePoolVolumeMount() (bool, error) {
s.volume.Name, s.pool.Name)
RBDFilesystem := s.getRBDFilesystem()
- RBDDevPath := getRBDDevPath(
- s.OSDPoolName,
- storagePoolVolumeTypeNameCustom,
- s.volume.Name)
volumeMntPoint := getStoragePoolVolumeMountPoint(s.pool.Name, s.volume.Name)
customMountLockID := getCustomMountLockID(s.pool.Name, s.volume.Name)
@@ -551,9 +539,14 @@ func (s *storageCeph) StoragePoolVolumeMount() (bool, error) {
lxdStorageOngoingOperationMap[customMountLockID] = make(chan bool)
lxdStorageMapLock.Unlock()
+ var ret int
var customerr error
ourMount := false
+ RBDDevPath := ""
if !shared.IsMountPoint(volumeMntPoint) {
+ RBDDevPath, ret = getRBDMappedDevPath(s.ClusterName, s.OSDPoolName,
+ storagePoolVolumeTypeNameCustom, s.volume.Name, true,
+ s.UserName)
mountFlags, mountOptions := lxdResolveMountoptions(s.getRBDMountOptions())
customerr = tryMount(
RBDDevPath,
@@ -571,7 +564,7 @@ func (s *storageCeph) StoragePoolVolumeMount() (bool, error) {
}
lxdStorageMapLock.Unlock()
- if customerr != nil {
+ if customerr != nil || ret < 0 {
logger.Errorf(`Failed to mount RBD storage volume "%s" on `+
`storage pool "%s": %s`, s.volume.Name, s.pool.Name,
customerr)
@@ -709,8 +702,8 @@ func (s *storageCeph) ContainerCreate(container container) error {
}
}()
- err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, containerName,
- storagePoolVolumeTypeNameContainer, s.UserName)
+ RBDDevPath, err := cephRBDVolumeMap(s.ClusterName, s.OSDPoolName,
+ containerName, storagePoolVolumeTypeNameContainer, s.UserName)
if err != nil {
logger.Errorf(`Failed to map RBD storage volume for `+
`container "%s" on storage pool "%s": %s`,
@@ -741,15 +734,6 @@ func (s *storageCeph) ContainerCreate(container container) error {
`for container "%s" on storage pool "%s"`, RBDFilesystem,
containerName, s.pool.Name)
- // get rbd device path
- RBDDevPath := getRBDDevPath(
- s.OSDPoolName,
- storagePoolVolumeTypeNameContainer,
- containerName)
- logger.Debugf(`Retrieved device path "%s" of RBD storage volume `+
- `for container "%s" on storage pool "%s"`, RBDDevPath,
- containerName, s.pool.Name)
-
msg, err := makeFSType(RBDDevPath, RBDFilesystem)
if err != nil {
logger.Errorf(`Failed to create filesystem type "%s" on `+
@@ -876,7 +860,7 @@ func (s *storageCeph) ContainerCreateFromImage(container container, fingerprint
}
}()
- err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, containerName,
+ _, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, containerName,
storagePoolVolumeTypeNameContainer, s.UserName)
if err != nil {
logger.Errorf(`Failed to map RBD storage volume for container `+
@@ -1293,7 +1277,7 @@ func (s *storageCeph) ContainerCopy(target container, source container,
// registered one above for the dummy volume we created.
// map the container's volume
- err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName,
+ _, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName,
targetContainerName, storagePoolVolumeTypeNameContainer,
s.UserName)
if err != nil {
@@ -1339,7 +1323,6 @@ func (s *storageCeph) ContainerMount(c container) (bool, error) {
logger.Debugf("Mounting RBD storage volume for container \"%s\" on storage pool \"%s\".", s.volume.Name, s.pool.Name)
RBDFilesystem := s.getRBDFilesystem()
- RBDDevPath := getRBDDevPath(s.OSDPoolName, storagePoolVolumeTypeNameContainer, name)
containerMntPoint := getContainerMountPoint(s.pool.Name, name)
if shared.IsSnapshot(name) {
containerMntPoint = getSnapshotMountPoint(s.pool.Name, name)
@@ -1361,11 +1344,17 @@ func (s *storageCeph) ContainerMount(c container) (bool, error) {
lxdStorageOngoingOperationMap[containerMountLockID] = make(chan bool)
lxdStorageMapLock.Unlock()
+ var ret int
var mounterr error
ourMount := false
+ RBDDevPath := ""
if !shared.IsMountPoint(containerMntPoint) {
+ RBDDevPath, ret = getRBDMappedDevPath(s.ClusterName,
+ s.OSDPoolName, storagePoolVolumeTypeNameContainer,
+ name, true, s.UserName)
mountFlags, mountOptions := lxdResolveMountoptions(s.getRBDMountOptions())
- mounterr = tryMount(RBDDevPath, containerMntPoint, RBDFilesystem, mountFlags, mountOptions)
+ mounterr = tryMount(RBDDevPath, containerMntPoint,
+ RBDFilesystem, mountFlags, mountOptions)
ourMount = true
}
@@ -1376,7 +1365,7 @@ func (s *storageCeph) ContainerMount(c container) (bool, error) {
}
lxdStorageMapLock.Unlock()
- if mounterr != nil {
+ if mounterr != nil || ret < 0 {
logger.Errorf("Failed to mount RBD storage volume for container \"%s\": %s", s.volume.Name, mounterr)
return false, mounterr
}
@@ -1464,8 +1453,8 @@ func (s *storageCeph) ContainerRename(c container, newName string) error {
return
}
- err := cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, oldName,
- storagePoolVolumeTypeNameContainer, s.UserName)
+ _, err := cephRBDVolumeMap(s.ClusterName, s.OSDPoolName,
+ oldName, storagePoolVolumeTypeNameContainer, s.UserName)
if err != nil {
logger.Warnf(`Failed to Map RBD storage volume `+
`for container "%s": %s`, oldName, err)
@@ -1499,7 +1488,7 @@ func (s *storageCeph) ContainerRename(c container, newName string) error {
}()
// map
- err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, newName,
+ _, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, newName,
storagePoolVolumeTypeNameContainer, s.UserName)
if err != nil {
logger.Errorf(`Failed to map RBD storage volume for `+
@@ -1934,8 +1923,8 @@ func (s *storageCeph) ContainerSnapshotStart(c container) (bool, error) {
}()
// map
- err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, cloneName,
- "snapshots", s.UserName)
+ RBDDevPath, err := cephRBDVolumeMap(s.ClusterName, s.OSDPoolName,
+ cloneName, "snapshots", s.UserName)
if err != nil {
logger.Errorf(`Failed to map RBD storage volume for `+
`container "%s" on storage pool "%s": %s`,
@@ -1961,7 +1950,6 @@ func (s *storageCeph) ContainerSnapshotStart(c container) (bool, error) {
containerMntPoint := getSnapshotMountPoint(s.pool.Name, containerName)
RBDFilesystem := s.getRBDFilesystem()
- RBDDevPath := getRBDDevPath(s.OSDPoolName, "snapshots", cloneName)
mountFlags, mountOptions := lxdResolveMountoptions(s.getRBDMountOptions())
err = tryMount(
RBDDevPath,
@@ -2123,8 +2111,9 @@ func (s *storageCeph) ImageCreate(fingerprint string) error {
}
}()
- err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName,
- fingerprint, storagePoolVolumeTypeNameImage, s.UserName)
+ RBDDevPath, err := cephRBDVolumeMap(s.ClusterName,
+ s.OSDPoolName, fingerprint,
+ storagePoolVolumeTypeNameImage, s.UserName)
if err != nil {
logger.Errorf(`Failed to map RBD storage volume for `+
`image "%s" on storage pool "%s": %s`,
@@ -2152,9 +2141,6 @@ func (s *storageCeph) ImageCreate(fingerprint string) error {
// get filesystem
RBDFilesystem := s.getRBDFilesystem()
- // get rbd device path
- RBDDevPath := getRBDDevPath(s.OSDPoolName,
- storagePoolVolumeTypeNameImage, fingerprint)
msg, err := makeFSType(RBDDevPath, RBDFilesystem)
if err != nil {
logger.Errorf(`Failed to create filesystem "%s" for RBD `+
@@ -2456,9 +2442,10 @@ func (s *storageCeph) ImageMount(fingerprint string) (bool, error) {
RBDFilesystem := s.getRBDFilesystem()
RBDMountOptions := s.getRBDMountOptions()
mountFlags, mountOptions := lxdResolveMountoptions(RBDMountOptions)
- RBDDevPath := getRBDDevPath(s.OSDPoolName, storagePoolVolumeTypeNameImage, fingerprint)
+ RBDDevPath, ret := getRBDMappedDevPath(s.ClusterName, s.OSDPoolName,
+ storagePoolVolumeTypeNameImage, fingerprint, true, s.UserName)
err := tryMount(RBDDevPath, imageMntPoint, RBDFilesystem, mountFlags, mountOptions)
- if err != nil {
+ if err != nil || ret < 0 {
logger.Errorf("Failed to mount RBD device %s onto %s: %s", RBDDevPath, imageMntPoint, err)
return false, err
}
diff --git a/lxd/storage_ceph_utils.go b/lxd/storage_ceph_utils.go
index 96785436c..1ce1e3f4f 100644
--- a/lxd/storage_ceph_utils.go
+++ b/lxd/storage_ceph_utils.go
@@ -3,8 +3,10 @@ package main
import (
"encoding/json"
"fmt"
+ "io/ioutil"
"os"
"os/exec"
+ "strconv"
"strings"
"syscall"
@@ -56,17 +58,6 @@ func cephOSDPoolDestroy(clusterName string, poolName string, userName string) er
return nil
}
-// getRBDDevPath constructs the path to a RBD block device.
-// Note that for this path to be valid the corresponding volume has to be mapped
-// first.
-func getRBDDevPath(poolName string, volumeType string, volumeName string) string {
- if volumeType == "" {
- return fmt.Sprintf("/dev/rbd/%s/%s", poolName, volumeName)
- }
-
- return fmt.Sprintf("/dev/rbd/%s/%s_%s", poolName, volumeType, volumeName)
-}
-
// cephRBDVolumeCreate creates an RBD storage volume.
// Note that the set of features is intentionally limited is intentionally
// limited by passing --image-feature explicitly. This is done to ensure that
@@ -129,8 +120,8 @@ func cephRBDVolumeDelete(clusterName string, poolName string, volumeName string,
// This will ensure that the RBD storage volume is accessible as a block device
// in the /dev directory and is therefore necessary in order to mount it.
func cephRBDVolumeMap(clusterName string, poolName string, volumeName string,
- volumeType string, userName string) error {
- _, err := shared.RunCommand(
+ volumeType string, userName string) (string, error) {
+ devPath, err := shared.RunCommand(
"rbd",
"--id", userName,
"--cluster", clusterName,
@@ -138,21 +129,10 @@ func cephRBDVolumeMap(clusterName string, poolName string, volumeName string,
"map",
fmt.Sprintf("%s_%s", volumeType, volumeName))
if err != nil {
- runError, ok := err.(shared.RunError)
- if ok {
- exitError, ok := runError.Err.(*exec.ExitError)
- if ok {
- waitStatus := exitError.Sys().(syscall.WaitStatus)
- if waitStatus.ExitStatus() == 22 {
- // EINVAL (already mapped)
- return nil
- }
- }
- }
- return err
+ return "", err
}
- return nil
+ return strings.TrimSpace(devPath), nil
}
// cephRBDVolumeUnmap unmaps a given RBD storage volume
@@ -694,7 +674,7 @@ func (s *storageCeph) copyWithoutSnapshotsFull(target container,
return err
}
- err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, targetContainerName,
+ _, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, targetContainerName,
storagePoolVolumeTypeNameContainer, s.UserName)
if err != nil {
logger.Errorf(`Failed to map RBD storage volume for image `+
@@ -788,8 +768,9 @@ func (s *storageCeph) copyWithoutSnapshotsSparse(target container,
return err
}
- err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, targetContainerName,
- storagePoolVolumeTypeNameContainer, s.UserName)
+ _, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName,
+ targetContainerName, storagePoolVolumeTypeNameContainer,
+ s.UserName)
if err != nil {
logger.Errorf(`Failed to map RBD storage volume for image `+
`"%s" on storage pool "%s": %s`, targetContainerName,
@@ -1394,3 +1375,90 @@ func parseClone(clone string) (string, string, string, error) {
return poolName, volumeType, volumeName, nil
}
+
+// getRBDMappedDevPath looks at sysfs to retrieve the device path
+// "/dev/rbd<idx>" for an RBD image. If it doesn't find it it will map it if
+// told to do so.
+func getRBDMappedDevPath(clusterName string, poolName string, volumeType string,
+ volumeName string, doMap bool, userName string) (string, int) {
+ files, err := ioutil.ReadDir("/sys/devices/rbd")
+ if err != nil {
+ if os.IsNotExist(err) {
+ return "", 0
+ }
+
+ return "", -1
+ }
+
+ for _, f := range files {
+ if !f.IsDir() {
+ continue
+ }
+
+ fName := f.Name()
+ idx, err := strconv.ParseUint(fName, 10, 64)
+ if err != nil {
+ continue
+ }
+
+ tmp := fmt.Sprintf("/sys/devices/rbd/%/pool", fName)
+ contents, err := ioutil.ReadFile(tmp)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+
+ return "", -1
+ }
+
+ detectedPoolName := strings.TrimSpace(string(contents))
+ if detectedPoolName != poolName {
+ continue
+ }
+
+ tmp = fmt.Sprintf("/sys/devices/rbd/%s/name", fName)
+ contents, err = ioutil.ReadFile(tmp)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+
+ return "", -1
+ }
+
+ typedVolumeName := fmt.Sprintf("%s_%s", volumeType, volumeName)
+ detectedVolumeName := strings.TrimSpace(string(contents))
+ if detectedVolumeName != typedVolumeName {
+ continue
+ }
+
+ tmp = fmt.Sprintf("/sys/devices/rbd/%s/snap", fName)
+ contents, err = ioutil.ReadFile(tmp)
+ if err != nil {
+ if os.IsNotExist(err) {
+ continue
+ }
+
+ return "", -1
+ }
+
+ detectedSnapName := strings.TrimSpace(string(contents))
+ if detectedSnapName != "-" {
+ continue
+ }
+
+ return fmt.Sprintf("/dev/rbd%d", idx), 1
+ }
+
+ if !doMap {
+ return "", 0
+ }
+
+ devPath, err := cephRBDVolumeMap(clusterName, poolName,
+ volumeName, volumeType, userName)
+ if err != nil {
+ return "", -1
+ }
+
+ return strings.TrimSpace(devPath), 2
+}
More information about the lxc-devel
mailing list