[lxc-devel] [lxd/master] btrfs storage driver
stgraber on Github
lxc-bot at linuxcontainers.org
Thu Dec 19 19:42:14 UTC 2019
A non-text attachment was scrubbed...
Name: not available
Type: text/x-mailbox
Size: 301 bytes
Desc: not available
URL: <http://lists.linuxcontainers.org/pipermail/lxc-devel/attachments/20191219/c9fab75f/attachment-0001.bin>
-------------- next part --------------
From b9f219ba53ea8a516e06a3ac4a095388ab80c691 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 19 Dec 2019 12:15:58 -0500
Subject: [PATCH 1/6] lxd/storage/drivers: Introduce vfsBackupVolume
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/storage/drivers/driver_common.go | 62 +++++++++++++++++++++++
lxd/storage/drivers/driver_dir_volumes.go | 51 +------------------
2 files changed, 64 insertions(+), 49 deletions(-)
diff --git a/lxd/storage/drivers/driver_common.go b/lxd/storage/drivers/driver_common.go
index cd85e90f98..f671f65c2c 100644
--- a/lxd/storage/drivers/driver_common.go
+++ b/lxd/storage/drivers/driver_common.go
@@ -283,3 +283,65 @@ func (d *common) vfsGetVolumeDiskPath(vol Volume) (string, error) {
return filepath.Join(vol.MountPath(), "root.img"), nil
}
+
+// vfsBackupVolume is a generic BackupVolume implementation for VFS-only drivers.
+func (d *common) vfsBackupVolume(vol Volume, targetPath string, snapshots bool, op *operations.Operation) error {
+ bwlimit := d.config["rsync.bwlimit"]
+
+ // Backups only implemented for containers currently.
+ if vol.volType != VolumeTypeContainer {
+ return ErrNotImplemented
+ }
+ // Handle snapshots.
+ if snapshots {
+ snapshotsPath := filepath.Join(targetPath, "snapshots")
+
+ // List the snapshots.
+ snapshots, err := vol.Snapshots(op)
+ if err != nil {
+ return err
+ }
+
+ // Create the snapshot path.
+ if len(snapshots) > 0 {
+ err = os.MkdirAll(snapshotsPath, 0711)
+ if err != nil {
+ return err
+ }
+ }
+
+ for _, snapshot := range snapshots {
+ _, snapName, _ := shared.InstanceGetParentAndSnapshotName(snapshot.Name())
+ target := filepath.Join(snapshotsPath, snapName)
+
+ // Copy the snapshot.
+ err = snapshot.MountTask(func(mountPath string, op *operations.Operation) error {
+ _, err := rsync.LocalCopy(mountPath, target, bwlimit, true)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ }, op)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ // Copy the parent volume itself.
+ target := filepath.Join(targetPath, "container")
+ err := vol.MountTask(func(mountPath string, op *operations.Operation) error {
+ _, err := rsync.LocalCopy(mountPath, target, bwlimit, true)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ }, op)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/lxd/storage/drivers/driver_dir_volumes.go b/lxd/storage/drivers/driver_dir_volumes.go
index bda0866ef1..28790c9209 100644
--- a/lxd/storage/drivers/driver_dir_volumes.go
+++ b/lxd/storage/drivers/driver_dir_volumes.go
@@ -4,7 +4,6 @@ import (
"fmt"
"io"
"os"
- "path/filepath"
"github.com/lxc/lxd/lxd/migration"
"github.com/lxc/lxd/lxd/operations"
@@ -293,54 +292,8 @@ func (d *dir) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs migr
// BackupVolume copies a volume (and optionally its snapshots) to a specified target path.
// This driver does not support optimized backups.
-func (d *dir) BackupVolume(vol Volume, targetPath string, _, snapshots bool, op *operations.Operation) error {
- bwlimit := d.config["rsync.bwlimit"]
-
- var parentVolDir string
-
- // Backups only implemented for containers currently.
- if vol.volType == VolumeTypeContainer {
- parentVolDir = "container"
- } else {
- return ErrNotImplemented
- }
-
- // Handle snapshots.
- if snapshots {
- snapshotsPath := filepath.Join(targetPath, "snapshots")
- snapshots, err := vol.Snapshots(op)
- if err != nil {
- return err
- }
-
- // Create the snapshot path.
- if len(snapshots) > 0 {
- err = os.MkdirAll(snapshotsPath, 0711)
- if err != nil {
- return err
- }
- }
-
- for _, snap := range snapshots {
- _, snapName, _ := shared.InstanceGetParentAndSnapshotName(snap.Name())
- target := filepath.Join(snapshotsPath, snapName)
-
- // Copy the snapshot.
- _, err := rsync.LocalCopy(snap.MountPath(), target, bwlimit, true)
- if err != nil {
- return fmt.Errorf("Failed to rsync: %s", err)
- }
- }
- }
-
- // Copy the parent volume itself.
- target := filepath.Join(targetPath, parentVolDir)
- _, err := rsync.LocalCopy(vol.MountPath(), target, bwlimit, true)
- if err != nil {
- return fmt.Errorf("Failed to rsync: %s", err)
- }
-
- return nil
+func (d *dir) BackupVolume(vol Volume, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
+ return d.vfsBackupVolume(vol, targetPath, snapshots, op)
}
// CreateVolumeSnapshot creates a snapshot of a volume.
From d2b0eb98fd6e34cc88759949fd54312dd07655e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 17 Dec 2019 12:48:20 -0500
Subject: [PATCH 2/6] lxd/storage/utils: Add fsUUID
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/storage/drivers/utils.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/lxd/storage/drivers/utils.go b/lxd/storage/drivers/utils.go
index a2230b18ab..b5fc345dfa 100644
--- a/lxd/storage/drivers/utils.go
+++ b/lxd/storage/drivers/utils.go
@@ -163,6 +163,10 @@ func TryUnmount(path string, flags int) error {
return nil
}
+func fsUUID(path string) (string, error) {
+ return shared.RunCommand("blkid", "-s", "UUID", "-o", "value", path)
+}
+
// GetPoolMountPath returns the mountpoint of the given pool.
// {LXD_DIR}/storage-pools/<pool>
func GetPoolMountPath(poolName string) string {
From 73a68159880f3075c3bd760ec1d3655164c2c473 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 17 Dec 2019 12:53:20 -0500
Subject: [PATCH 3/6] lxd/storage/utils: Add tryExists
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/storage/drivers/utils.go | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/lxd/storage/drivers/utils.go b/lxd/storage/drivers/utils.go
index b5fc345dfa..fb84da724c 100644
--- a/lxd/storage/drivers/utils.go
+++ b/lxd/storage/drivers/utils.go
@@ -163,6 +163,19 @@ func TryUnmount(path string, flags int) error {
return nil
}
+func tryExists(path string) bool {
+ // Attempt 20 checks over 10s
+ for i := 0; i < 20; i++ {
+ if shared.PathExists(path) {
+ return true
+ }
+
+ time.Sleep(500 * time.Millisecond)
+ }
+
+ return false
+}
+
func fsUUID(path string) (string, error) {
return shared.RunCommand("blkid", "-s", "UUID", "-o", "value", path)
}
From c9bc049c47e6d2c6564edbd9b63d5ebf9a6049f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Tue, 17 Dec 2019 12:54:13 -0500
Subject: [PATCH 4/6] lxd/storage/utils: Add hasFilesystem
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/storage/drivers/utils.go | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/lxd/storage/drivers/utils.go b/lxd/storage/drivers/utils.go
index fb84da724c..5d46ea617f 100644
--- a/lxd/storage/drivers/utils.go
+++ b/lxd/storage/drivers/utils.go
@@ -180,6 +180,21 @@ func fsUUID(path string) (string, error) {
return shared.RunCommand("blkid", "-s", "UUID", "-o", "value", path)
}
+func hasFilesystem(path string, fsType int64) bool {
+ fs := unix.Statfs_t{}
+
+ err := unix.Statfs(path, &fs)
+ if err != nil {
+ return false
+ }
+
+ if fs.Type != fsType {
+ return false
+ }
+
+ return true
+}
+
// GetPoolMountPath returns the mountpoint of the given pool.
// {LXD_DIR}/storage-pools/<pool>
func GetPoolMountPath(poolName string) string {
From 45c55d5ced24851149e8e128e23d90477f473975 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.hipp at canonical.com>
Date: Mon, 18 Nov 2019 15:24:41 +0100
Subject: [PATCH 5/6] lxd/storage/drivers: Add btrfs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This adds the btrfs storage driver.
Signed-off-by: Thomas Hipp <thomas.hipp at canonical.com>
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
lxd/storage/drivers/driver_btrfs.go | 361 ++++++++++
lxd/storage/drivers/driver_btrfs_utils.go | 374 ++++++++++
lxd/storage/drivers/driver_btrfs_volumes.go | 759 ++++++++++++++++++++
lxd/storage/drivers/load.go | 1 +
4 files changed, 1495 insertions(+)
create mode 100644 lxd/storage/drivers/driver_btrfs.go
create mode 100644 lxd/storage/drivers/driver_btrfs_utils.go
create mode 100644 lxd/storage/drivers/driver_btrfs_volumes.go
diff --git a/lxd/storage/drivers/driver_btrfs.go b/lxd/storage/drivers/driver_btrfs.go
new file mode 100644
index 0000000000..cbc1016628
--- /dev/null
+++ b/lxd/storage/drivers/driver_btrfs.go
@@ -0,0 +1,361 @@
+package drivers
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ "golang.org/x/sys/unix"
+
+ "github.com/lxc/lxd/lxd/migration"
+ "github.com/lxc/lxd/lxd/operations"
+ "github.com/lxc/lxd/lxd/util"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/api"
+ "github.com/lxc/lxd/shared/logger"
+ "github.com/lxc/lxd/shared/units"
+)
+
+var btrfsVersion string
+var btrfsLoaded bool
+
+type btrfs struct {
+ common
+}
+
+// load is used to run one-time action per-driver rather than per-pool.
+func (d *btrfs) load() error {
+ if btrfsLoaded {
+ return nil
+ }
+
+ // Validate the required binaries.
+ for _, tool := range []string{"btrfs"} {
+ _, err := exec.LookPath(tool)
+ if err != nil {
+ return fmt.Errorf("Required tool '%s' is missing", tool)
+ }
+ }
+
+ // Detect and record the version.
+ if btrfsVersion == "" {
+ out, err := shared.RunCommand("btrfs", "version")
+ if err != nil {
+ return err
+ }
+
+ count, err := fmt.Sscanf(strings.SplitN(out, " ", 2)[1], "v%s\n", &btrfsVersion)
+ if err != nil || count != 1 {
+ return fmt.Errorf("The 'btrfs' tool isn't working properly")
+ }
+ }
+
+ btrfsLoaded = true
+ return nil
+}
+
+// Info returns info about the driver and its environment.
+func (d *btrfs) Info() Info {
+ return Info{
+ Name: "btrfs",
+ Version: btrfsVersion,
+ OptimizedImages: true,
+ PreservesInodes: !d.state.OS.RunningInUserNS,
+ Remote: false,
+ VolumeTypes: []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM},
+ BlockBacking: false,
+ RunningQuotaResize: true,
+ RunningSnapshotFreeze: false,
+ }
+}
+
+// Create is called during pool creation and is effectively using an empty driver struct.
+// WARNING: The Create() function cannot rely on any of the struct attributes being set.
+func (d *btrfs) Create() error {
+ // Store the provided source as we are likely to be mangling it.
+ d.config["volatile.initial_source"] = d.config["source"]
+
+ loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
+ if d.config["source"] == "" || d.config["source"] == loopPath {
+ // Create a loop based pool.
+ d.config["source"] = loopPath
+
+ // Create the loop file itself.
+ size, err := units.ParseByteSizeString(d.config["size"])
+ if err != nil {
+ return err
+ }
+
+ err = createSparseFile(d.config["source"], size)
+ if err != nil {
+ return fmt.Errorf("Failed to create the sparse file: %v", err)
+ }
+
+ // Format the file.
+ _, err = makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
+ if err != nil {
+ return fmt.Errorf("Failed to format sparse file: %v", err)
+ }
+ } else if shared.IsBlockdevPath(d.config["source"]) {
+ // Format the block device.
+ _, err := makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
+ if err != nil {
+ return fmt.Errorf("Failed to format block device: %v", err)
+ }
+
+ // Record the UUID as the source.
+ devUUID, err := fsUUID(d.config["source"])
+ if err != nil {
+ return err
+ }
+
+ // Confirm that the symlink is appearing (give it 10s).
+ if tryExists(fmt.Sprintf("/dev/disk/by-uuid/%s", devUUID)) {
+ // Override the config to use the UUID.
+ d.config["source"] = devUUID
+ }
+ } else if d.config["source"] != "" {
+ hostPath := shared.HostPath(d.config["source"])
+ if d.isSubvolume(hostPath) {
+ // Existing btrfs subvolume.
+ subvols, err := d.getSubvolumes(hostPath)
+ if err != nil {
+ return fmt.Errorf("Could not determine if existing btrfs subvolume is empty: %v", err)
+ }
+
+ // Check that the provided subvolume is empty.
+ if len(subvols) > 0 {
+ return fmt.Errorf("Requested btrfs subvolume exists but is not empty")
+ }
+ } else {
+ // New btrfs subvolume on existing btrfs filesystem.
+ cleanSource := filepath.Clean(hostPath)
+ lxdDir := shared.VarPath()
+
+ if shared.PathExists(hostPath) && !hasFilesystem(hostPath, util.FilesystemSuperMagicBtrfs) {
+ return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
+ } else if strings.HasPrefix(cleanSource, lxdDir) {
+ if cleanSource != GetPoolMountPath(d.name) {
+ return fmt.Errorf("Only allowed source path under %s is %s", shared.VarPath(), GetPoolMountPath(d.name))
+ } else if !hasFilesystem(shared.VarPath("storage-pools"), util.FilesystemSuperMagicBtrfs) {
+ return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
+ }
+
+ // Delete the current directory to replace by subvolume.
+ err := os.Remove(cleanSource)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Create the subvolume.
+ _, err := shared.RunCommand("btrfs", "subvolume", "create", hostPath)
+ if err != nil {
+ return err
+ }
+ }
+ } else {
+ return fmt.Errorf("Invalid \"source\" property")
+ }
+
+ return nil
+}
+
+// Delete removes the storage pool from the storage device.
+func (d *btrfs) Delete(op *operations.Operation) error {
+ // If the user completely destroyed it, call it done.
+ if !shared.PathExists(GetPoolMountPath(d.name)) {
+ return nil
+ }
+
+ // Delete potential intermediate btrfs subvolumes.
+ for _, volType := range d.Info().VolumeTypes {
+ for _, dir := range BaseDirectories[volType] {
+ path := filepath.Join(GetPoolMountPath(d.name), dir)
+ if !shared.PathExists(path) {
+ continue
+ }
+
+ if !d.isSubvolume(path) {
+ continue
+ }
+
+ err := d.deleteSubvolume(path, true)
+ if err != nil {
+ return fmt.Errorf("Could not delete btrfs subvolume: %s", path)
+ }
+ }
+ }
+
+ // On delete, wipe everything in the directory.
+ err := wipeDirectory(GetPoolMountPath(d.name))
+ if err != nil {
+ return err
+ }
+
+ // Unmount the path.
+ _, err = d.Unmount()
+ if err != nil {
+ return err
+ }
+
+ // If the pool path is a subvolume itself, delete it.
+ if d.isSubvolume(GetPoolMountPath(d.name)) {
+ err := d.deleteSubvolume(GetPoolMountPath(d.name), false)
+ if err != nil {
+ return err
+ }
+
+ // And re-create as an empty directory to make the backend happy.
+ err = os.Mkdir(GetPoolMountPath(d.name), 0700)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Delete any loop file we may have used.
+ loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
+ if shared.PathExists(loopPath) {
+ err = os.Remove(loopPath)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// Validate checks that all provide keys are supported and that no conflicting or missing configuration is present.
+func (d *btrfs) Validate(config map[string]string) error {
+ return nil
+}
+
+// Update applies any driver changes required from a configuration change.
+func (d *btrfs) Update(changedConfig map[string]string) error {
+ // We only care about btrfs.mount_options.
+ val, ok := changedConfig["btrfs.mount_options"]
+ if !ok {
+ return nil
+ }
+
+ // Trigger a re-mount.
+ d.config["btrfs.mount_options"] = val
+ mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())
+ mntFlags |= unix.MS_REMOUNT
+
+ err := TryMount("", GetPoolMountPath(d.name), "none", mntFlags, mntOptions)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// Mount mounts the storage pool.
+func (d *btrfs) Mount() (bool, error) {
+ // Check if already mounted.
+ if shared.IsMountPoint(GetPoolMountPath(d.name)) {
+ logger.Errorf("here: %v", d.name)
+ return false, nil
+ }
+
+ // Setup mount options.
+ loopPath := filepath.Join(shared.VarPath("disks"), fmt.Sprintf("%s.img", d.name))
+ mntSrc := ""
+ mntDst := GetPoolMountPath(d.name)
+ mntFilesystem := "btrfs"
+ if d.config["source"] == loopPath {
+ // Bring up the loop device.
+ loopF, err := PrepareLoopDev(d.config["source"], LoFlagsAutoclear)
+ if err != nil {
+ return false, err
+ }
+ defer loopF.Close()
+
+ mntSrc = loopF.Name()
+ } else if filepath.IsAbs(d.config["source"]) {
+ // Bring up an existing device or path.
+ mntSrc = shared.HostPath(d.config["source"])
+
+ if !shared.IsBlockdevPath(mntSrc) {
+ mntFilesystem = "none"
+
+ if !hasFilesystem(mntSrc, util.FilesystemSuperMagicBtrfs) {
+ return false, fmt.Errorf("Source path '%s' isn't btrfs", mntSrc)
+ }
+ }
+ } else {
+ // Mount using UUID.
+ mntSrc = fmt.Sprintf("/dev/disk/by-uuid/%s", d.config["source"])
+ }
+
+ // Get the custom mount flags/options.
+ mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())
+
+ // Handle bind-mounts first.
+ if mntFilesystem == "none" {
+ // Setup the bind-mount itself.
+ err := TryMount(mntSrc, mntDst, mntFilesystem, unix.MS_BIND, "")
+ if err != nil {
+ return false, err
+ }
+
+ // Now apply the custom options.
+ mntFlags |= unix.MS_REMOUNT
+ err = TryMount("", mntDst, mntFilesystem, mntFlags, mntOptions)
+ if err != nil {
+ return false, err
+ }
+
+ return true, nil
+ }
+
+ // Handle traditional mounts.
+ err := TryMount(mntSrc, mntDst, mntFilesystem, mntFlags, mntOptions)
+ if err != nil {
+ return false, err
+ }
+
+ return true, nil
+}
+
+// Unmount unmounts the storage pool.
+func (d *btrfs) Unmount() (bool, error) {
+ return forceUnmount(GetPoolMountPath(d.name))
+}
+
+// GetResources returns the pool resource usage information.
+func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) {
+ return d.vfsGetResources()
+}
+
+// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
+func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
+ if contentType != ContentTypeFS {
+ return nil
+ }
+
+ // When performing a refresh, always use rsync. Using btrfs send/receive
+ // here doesn't make sense since it would need to send everything again
+ // which defeats the purpose of a refresh.
+ if refresh {
+ return []migration.Type{
+ {
+ FSType: migration.MigrationFSType_RSYNC,
+ Features: []string{"xattrs", "delete", "compress", "bidirectional"},
+ },
+ }
+ }
+
+ return []migration.Type{
+ {
+ FSType: migration.MigrationFSType_BTRFS,
+ },
+ {
+ FSType: migration.MigrationFSType_RSYNC,
+ Features: []string{"xattrs", "delete", "compress", "bidirectional"},
+ },
+ }
+}
diff --git a/lxd/storage/drivers/driver_btrfs_utils.go b/lxd/storage/drivers/driver_btrfs_utils.go
new file mode 100644
index 0000000000..5f759dd3c6
--- /dev/null
+++ b/lxd/storage/drivers/driver_btrfs_utils.go
@@ -0,0 +1,374 @@
+package drivers
+
+import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strconv"
+ "strings"
+
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/ioprogress"
+ "github.com/lxc/lxd/shared/logger"
+ "golang.org/x/sys/unix"
+)
+
+// Errors
+var errBtrfsNoQuota = fmt.Errorf("Quotas disabled on filesystem")
+var errBtrfsNoQGroup = fmt.Errorf("Unable to find quota group")
+
+func (d *btrfs) getMountOptions() string {
+ // Allow overriding the default options.
+ if d.config["btrfs.mount_options"] != "" {
+ return d.config["btrfs.mount_options"]
+ }
+
+ return "user_subvol_rm_allowed"
+}
+
+func (d *btrfs) isSubvolume(path string) bool {
+ // Stat the path.
+ fs := unix.Stat_t{}
+ err := unix.Lstat(path, &fs)
+ if err != nil {
+ return false
+ }
+
+ // Check if BTRFS_FIRST_FREE_OBJECTID is the inode number.
+ if fs.Ino != 256 {
+ return false
+ }
+
+ return true
+}
+
+func (d *btrfs) getSubvolumes(path string) ([]string, error) {
+ result := []string{}
+
+ // Make sure the path has a trailing slash.
+ if !strings.HasSuffix(path, "/") {
+ path = path + "/"
+ }
+
+ // Walk through the entire tree looking for subvolumes.
+ err := filepath.Walk(path, func(fpath string, fi os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+
+ // Ignore the base path.
+ if strings.TrimRight(fpath, "/") == strings.TrimRight(path, "/") {
+ return nil
+ }
+
+ // Subvolumes can only be directories.
+ if !fi.IsDir() {
+ return nil
+ }
+
+ // Check if a subvolume.
+ if d.isSubvolume(fpath) {
+ result = append(result, strings.TrimPrefix(fpath, path))
+ }
+
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ return result, nil
+}
+
+func (d *btrfs) snapshotSubvolume(path string, dest string, readonly bool, recursion bool) error {
+ // Single subvolume deletion.
+ snapshot := func(path string, dest string) error {
+ if readonly && !d.state.OS.RunningInUserNS {
+ _, err := shared.RunCommand("btrfs", "subvolume", "snapshot", "-r", path, dest)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ _, err := shared.RunCommand("btrfs", "subvolume", "snapshot", path, dest)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ // Now snapshot all subvolumes of the root.
+ if recursion {
+ // Get the subvolumes list.
+ subsubvols, err := d.getSubvolumes(path)
+ if err != nil {
+ return err
+ }
+ sort.Sort(sort.StringSlice(subsubvols))
+
+ if len(subsubvols) > 0 && readonly {
+ // Creating subvolumes requires the parent to be writable.
+ readonly = false
+ }
+
+ // First snapshot the root.
+ err = snapshot(path, dest)
+ if err != nil {
+ return err
+ }
+
+ for _, subsubvol := range subsubvols {
+ // Clear the target for the subvol to use.
+ os.Remove(filepath.Join(dest, subsubvol))
+
+ err := snapshot(filepath.Join(path, subsubvol), filepath.Join(dest, subsubvol))
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+ }
+
+ // Handle standalone volume.
+ err := snapshot(path, dest)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (d *btrfs) deleteSubvolume(path string, recursion bool) error {
+ // Single subvolume deletion.
+ destroy := func(path string) error {
+ // Attempt (but don't fail on) to delete any qgroup on the subvolume.
+ qgroup, _, err := d.getQGroup(path)
+ if err == nil {
+ shared.RunCommand("btrfs", "qgroup", "destroy", qgroup, path)
+ }
+
+ // Attempt to make the subvolume writable.
+ shared.RunCommand("btrfs", "property", "set", path, "ro", "false")
+
+ // Delete the subvolume itself.
+ _, err = shared.RunCommand("btrfs", "subvolume", "delete", path)
+
+ return err
+ }
+
+ // Delete subsubvols.
+ if recursion {
+ // Get the subvolumes list.
+ subsubvols, err := d.getSubvolumes(path)
+ if err != nil {
+ return err
+ }
+ sort.Sort(sort.Reverse(sort.StringSlice(subsubvols)))
+
+ for _, subsubvol := range subsubvols {
+ err := destroy(filepath.Join(path, subsubvol))
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ // Delete the subvol itself.
+ err := destroy(path)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (d *btrfs) getQGroup(path string) (string, int64, error) {
+ // Try to get the qgroup details.
+ output, err := shared.RunCommand("btrfs", "qgroup", "show", "-e", "-f", path)
+ if err != nil {
+ return "", -1, errBtrfsNoQuota
+ }
+
+ // Parse to extract the qgroup identifier.
+ var qgroup string
+ usage := int64(-1)
+ for _, line := range strings.Split(output, "\n") {
+ if line == "" || strings.HasPrefix(line, "qgroupid") || strings.HasPrefix(line, "---") {
+ continue
+ }
+
+ fields := strings.Fields(line)
+ if len(fields) != 4 {
+ continue
+ }
+
+ qgroup = fields[0]
+ val, err := strconv.ParseInt(fields[2], 10, 64)
+ if err == nil {
+ usage = val
+ }
+
+ break
+ }
+
+ if qgroup == "" {
+ return "", -1, errBtrfsNoQGroup
+ }
+
+ return qgroup, usage, nil
+}
+
+func (d *btrfs) sendSubvolume(path string, parent string, conn io.ReadWriteCloser, tracker *ioprogress.ProgressTracker) error {
+ // Assemble btrfs send command.
+ args := []string{"send"}
+ if parent != "" {
+ args = append(args, "-p", parent)
+ }
+ args = append(args, path)
+ cmd := exec.Command("btrfs", args...)
+
+ // Prepare stdout/stderr.
+ stdout, err := cmd.StdoutPipe()
+ if err != nil {
+ return err
+ }
+
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return err
+ }
+
+ // Setup progress tracker.
+ stdoutPipe := stdout
+ if tracker != nil {
+ stdoutPipe = &ioprogress.ProgressReader{
+ ReadCloser: stdout,
+ Tracker: tracker,
+ }
+ }
+
+ // Forward any output on stdout.
+ chStdoutPipe := make(chan error, 1)
+ go func() {
+ _, err := io.Copy(conn, stdoutPipe)
+ chStdoutPipe <- err
+ conn.Close()
+ }()
+
+ // Run the command.
+ err = cmd.Start()
+ if err != nil {
+ return err
+ }
+
+ // Read any error.
+ output, err := ioutil.ReadAll(stderr)
+ if err != nil {
+ logger.Errorf("Problem reading btrfs send stderr: %s", err)
+ }
+
+ // Handle errors.
+ errs := []error{}
+ chStdoutPipeErr := <-chStdoutPipe
+
+ err = cmd.Wait()
+ if err != nil {
+ errs = append(errs, err)
+
+ if chStdoutPipeErr != nil {
+ errs = append(errs, chStdoutPipeErr)
+ }
+ }
+
+ if len(errs) > 0 {
+ return fmt.Errorf("Btrfs send failed: %v (%s)", errs, string(output))
+ }
+
+ return nil
+}
+
+func (d *btrfs) receiveSubvolume(path string, targetPath string, conn io.ReadWriteCloser, writeWrapper func(io.WriteCloser) io.WriteCloser) error {
+ // Assemble btrfs send command.
+ cmd := exec.Command("btrfs", "receive", "-e", path)
+
+ // Prepare stdin/stderr.
+ stdin, err := cmd.StdinPipe()
+ if err != nil {
+ return err
+ }
+
+ stderr, err := cmd.StderrPipe()
+ if err != nil {
+ return err
+ }
+
+ // Forward input through stdin.
+ chCopyConn := make(chan error, 1)
+ go func() {
+ _, err = io.Copy(stdin, conn)
+ stdin.Close()
+ chCopyConn <- err
+ }()
+
+ // Run the command.
+ err = cmd.Start()
+ if err != nil {
+ return err
+ }
+
+ // Read any error.
+ output, err := ioutil.ReadAll(stderr)
+ if err != nil {
+ logger.Debugf("Problem reading btrfs receive stderr %s", err)
+ }
+
+ // Handle errors.
+ errs := []error{}
+ chCopyConnErr := <-chCopyConn
+
+ err = cmd.Wait()
+ if err != nil {
+ errs = append(errs, err)
+
+ if chCopyConnErr != nil {
+ errs = append(errs, chCopyConnErr)
+ }
+ }
+
+ if len(errs) > 0 {
+ return fmt.Errorf("Problem with btrfs receive: (%v) %s", errs, string(output))
+ }
+
+ // If we receive and target paths match, we're done.
+ if path == targetPath {
+ return nil
+ }
+
+ // Handle older LXD versions.
+ receivedSnapshot := fmt.Sprintf("%s/.migration-send", path)
+ if !shared.PathExists(receivedSnapshot) {
+ receivedSnapshot = fmt.Sprintf("%s/.root", path)
+ }
+
+ // Mark the received subvolume writable.
+ _, err = shared.RunCommand("btrfs", "property", "set", "-ts", receivedSnapshot, "ro", "false")
+ if err != nil {
+ return err
+ }
+
+ // And move it to the target path.
+ err = os.Rename(receivedSnapshot, targetPath)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/lxd/storage/drivers/driver_btrfs_volumes.go b/lxd/storage/drivers/driver_btrfs_volumes.go
new file mode 100644
index 0000000000..ba4264db9c
--- /dev/null
+++ b/lxd/storage/drivers/driver_btrfs_volumes.go
@@ -0,0 +1,759 @@
+package drivers
+
+import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/lxc/lxd/lxd/migration"
+ "github.com/lxc/lxd/lxd/operations"
+ "github.com/lxc/lxd/shared"
+ "github.com/lxc/lxd/shared/ioprogress"
+ "github.com/lxc/lxd/shared/units"
+)
+
+// CreateVolume creates an empty volume and can optionally fill it by executing the supplied filler function.
+func (d *btrfs) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Operation) error {
+ volPath := vol.MountPath()
+
+ // Create the volume itself.
+ _, err := shared.RunCommand("btrfs", "subvolume", "create", volPath)
+ if err != nil {
+ return err
+ }
+
+ // Setup revert.
+ revertPath := true
+ defer func() {
+ if revertPath {
+ d.deleteSubvolume(volPath, false)
+ }
+ }()
+
+ // Create sparse loopback file if volume is block.
+ rootBlockPath := ""
+ if vol.contentType == ContentTypeBlock {
+ // We expect the filler to copy the VM image into this path.
+ rootBlockPath, err = d.GetVolumeDiskPath(vol)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Run the volume filler function if supplied.
+ if filler != nil && filler.Fill != nil {
+ err = filler.Fill(volPath, rootBlockPath)
+ if err != nil {
+ return err
+ }
+ }
+
+ // If we are creating a block volume, resize it to the requested size or the default.
+ // We expect the filler function to have converted the qcow2 image to raw into the rootBlockPath.
+ if vol.contentType == ContentTypeBlock {
+ err := ensureVolumeBlockFile(vol, rootBlockPath)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Tweak any permissions that need tweaking.
+ err = vol.EnsureMountPath()
+ if err != nil {
+ return err
+ }
+
+ // Attempt to mark image read-only.
+ if vol.volType == VolumeTypeImage {
+ _, err = shared.RunCommand("btrfs", "property", "set", volPath, "ro", "true")
+ if err != nil && !d.state.OS.RunningInUserNS {
+ return err
+ }
+ }
+
+ revertPath = false
+ return nil
+}
+
+// CreateVolumeFromBackup restores a backup tarball onto the storage device.
+func (d *btrfs) CreateVolumeFromBackup(vol Volume, snapshots []string, srcData io.ReadSeeker, optimized bool, op *operations.Operation) (func(vol Volume) error, func(), error) {
+ // Handle the non-optimized tarballs through the generic unpacker.
+ if !optimized {
+ return genericBackupUnpack(d, vol, snapshots, srcData, op)
+ }
+
+ // Now deal with the binary btrfs backups.
+ revert := true
+
+ // Define a revert function that will be used both to revert if an error occurs inside this
+ // function but also return it for use from the calling functions if no error internally.
+ revertHook := func() {
+ for _, snapName := range snapshots {
+ fullSnapshotName := GetSnapshotVolumeName(vol.name, snapName)
+ snapVol := NewVolume(d, d.name, vol.volType, vol.contentType, fullSnapshotName, vol.config)
+ d.DeleteVolumeSnapshot(snapVol, op)
+ }
+
+ // And lastly the main volume.
+ d.DeleteVolume(vol, op)
+ }
+
+ // Only execute the revert function if we have had an error internally and revert is true.
+ defer func() {
+ if revert {
+ revertHook()
+ }
+ }()
+
+ // Create a temporary directory to unpack the backup into.
+ unpackDir, err := ioutil.TempDir(GetVolumeMountPath(d.name, vol.volType, ""), vol.name)
+ if err != nil {
+ return nil, nil, err
+ }
+ defer os.RemoveAll(unpackDir)
+
+ err = os.Chmod(unpackDir, 0100)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ // Find the compression algorithm used for backup source data.
+ srcData.Seek(0, 0)
+ tarArgs, _, _, err := shared.DetectCompressionFile(srcData)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ // Prepare tar arguments.
+ args := append(tarArgs, []string{
+ "-",
+ "--strip-components=1",
+ "-C", unpackDir, "backup",
+ }...)
+
+ // Unpack the backup.
+ srcData.Seek(0, 0)
+ err = shared.RunCommandWithFds(srcData, nil, "tar", args...)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ if len(snapshots) > 0 {
+ // Create new snapshots directory.
+ err := createParentSnapshotDirIfMissing(d.name, vol.volType, vol.name)
+ if err != nil {
+ return nil, nil, err
+ }
+ }
+
+ // Restore backups from oldest to newest.
+ snapshotsDir := GetVolumeSnapshotDir(d.name, vol.volType, vol.name)
+ for _, snapName := range snapshots {
+ // Open the backup.
+ feeder, err := os.Open(filepath.Join(unpackDir, "snapshots", fmt.Sprintf("%s.bin", snapName)))
+ if err != nil {
+ return nil, nil, err
+ }
+ defer feeder.Close()
+
+ // Extract the backup.
+ err = shared.RunCommandWithFds(feeder, nil, "btrfs", "receive", "-e", snapshotsDir)
+ if err != nil {
+ return nil, nil, err
+ }
+ }
+
+ // Open the backup.
+ feeder, err := os.Open(filepath.Join(unpackDir, "container.bin"))
+ if err != nil {
+ return nil, nil, err
+ }
+ defer feeder.Close()
+
+ // Extrack the backup.
+ err = shared.RunCommandWithFds(feeder, nil, "btrfs", "receive", "-e", unpackDir)
+ if err != nil {
+ return nil, nil, err
+ }
+ defer d.deleteSubvolume(filepath.Join(unpackDir, ".backup"), true)
+
+ // Re-create the writable subvolume.
+ err = d.snapshotSubvolume(filepath.Join(unpackDir, ".backup"), vol.MountPath(), false, false)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ revert = false
+
+ return nil, revertHook, nil
+}
+
+// CreateVolumeFromCopy provides same-pool volume copying functionality.
+func (d *btrfs) CreateVolumeFromCopy(vol Volume, srcVol Volume, copySnapshots bool, op *operations.Operation) error {
+ // Recursively copy the main volume.
+ err := d.snapshotSubvolume(srcVol.MountPath(), vol.MountPath(), false, true)
+ if err != nil {
+ return err
+ }
+
+ // Fixup permissions.
+ err = vol.EnsureMountPath()
+ if err != nil {
+ return err
+ }
+
+ // If we're not copying any snapshots, we're done here.
+ if !copySnapshots || srcVol.IsSnapshot() {
+ return nil
+ }
+
+ // Get the list of snapshots.
+ snapshots, err := d.VolumeSnapshots(srcVol, op)
+ if err != nil {
+ return err
+ }
+
+ // If no snapshots, we're done here.
+ if len(snapshots) == 0 {
+ return nil
+ }
+
+ // Create the parent directory.
+ err = createParentSnapshotDirIfMissing(d.name, vol.volType, vol.name)
+ if err != nil {
+ return err
+ }
+
+ // Copy the snapshots.
+ for _, snapName := range snapshots {
+ srcSnapshot := GetVolumeMountPath(d.name, srcVol.volType, GetSnapshotVolumeName(srcVol.name, snapName))
+ dstSnapshot := GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, snapName))
+
+ err = d.snapshotSubvolume(srcSnapshot, dstSnapshot, true, false)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// CreateVolumeFromMigration creates a volume being sent via a migration.
+func (d *btrfs) CreateVolumeFromMigration(vol Volume, conn io.ReadWriteCloser, volTargetArgs migration.VolumeTargetArgs, preFiller *VolumeFiller, op *operations.Operation) error {
+ if vol.contentType != ContentTypeFS {
+ return fmt.Errorf("Content type not supported")
+ }
+
+ // Handle simple rsync through generic.
+ if volTargetArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC {
+ return genericCreateVolumeFromMigration(d, nil, vol, conn, volTargetArgs, preFiller, op)
+ } else if volTargetArgs.MigrationType.FSType != migration.MigrationFSType_BTRFS {
+ return fmt.Errorf("Migration type not supported")
+ }
+
+ // Handle btrfs send/receive migration.
+ if len(volTargetArgs.Snapshots) > 0 {
+ snapshotsDir := GetVolumeSnapshotDir(d.name, vol.volType, vol.name)
+
+ // Create the parent directory.
+ err := createParentSnapshotDirIfMissing(d.name, vol.volType, vol.name)
+ if err != nil {
+ return err
+ }
+
+ // Transfer the snapshots.
+ for _, snapName := range volTargetArgs.Snapshots {
+ fullSnapshotName := GetSnapshotVolumeName(vol.name, snapName)
+ wrapper := migration.ProgressWriter(op, "fs_progress", fullSnapshotName)
+
+ err = d.receiveSubvolume(snapshotsDir, snapshotsDir, conn, wrapper)
+ if err != nil {
+ return err
+ }
+ }
+ }
+
+ // Get instances directory (e.g. /var/lib/lxd/storage-pools/btrfs/containers).
+ instancesPath := GetVolumeMountPath(d.name, vol.volType, "")
+
+ // Create a temporary directory which will act as the parent directory of the received ro snapshot.
+ tmpVolumesMountPoint, err := ioutil.TempDir(instancesPath, vol.name)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpVolumesMountPoint)
+
+ err = os.Chmod(tmpVolumesMountPoint, 0100)
+ if err != nil {
+ return err
+ }
+
+ wrapper := migration.ProgressWriter(op, "fs_progress", vol.name)
+ err = d.receiveSubvolume(tmpVolumesMountPoint, vol.MountPath(), conn, wrapper)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// RefreshVolume provides same-pool volume and specific snapshots syncing functionality.
+func (d *btrfs) RefreshVolume(vol Volume, srcVol Volume, srcSnapshots []Volume, op *operations.Operation) error {
+ return genericCopyVolume(d, nil, vol, srcVol, srcSnapshots, op)
+}
+
+// DeleteVolume deletes a volume of the storage device. If any snapshots of the volume remain then
+// this function will return an error.
+func (d *btrfs) DeleteVolume(vol Volume, op *operations.Operation) error {
+ // Check that we don't have snapshots.
+ snapshots, err := d.VolumeSnapshots(vol, op)
+ if err != nil {
+ return err
+ }
+
+ if len(snapshots) > 0 {
+ return fmt.Errorf("Cannot remove a volume that has snapshots")
+ }
+
+ // If the volume doesn't exist, then nothing more to do.
+ volPath := GetVolumeMountPath(d.name, vol.volType, vol.name)
+ if !shared.PathExists(volPath) {
+ return nil
+ }
+
+ // Delete the volume (and any subvolumes).
+ err = d.deleteSubvolume(volPath, true)
+ if err != nil {
+ return err
+ }
+
+ // Although the volume snapshot directory should already be removed, lets remove it here
+ // to just in case the top-level directory is left.
+ err = deleteParentSnapshotDirIfEmpty(d.name, vol.volType, vol.name)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// HasVolume indicates whether a specific volume exists on the storage pool.
+func (d *btrfs) HasVolume(vol Volume) bool {
+ return d.vfsHasVolume(vol)
+}
+
+// ValidateVolume validates the supplied volume config.
+func (d *btrfs) ValidateVolume(vol Volume, removeUnknownKeys bool) error {
+ return d.validateVolume(vol, nil, removeUnknownKeys)
+}
+
+// UpdateVolume applies config changes to the volume.
+func (d *btrfs) UpdateVolume(vol Volume, changedConfig map[string]string) error {
+ if vol.contentType != ContentTypeFS {
+ return fmt.Errorf("Content type not supported")
+ }
+
+ if vol.volType != VolumeTypeCustom {
+ return fmt.Errorf("Volume type not supported")
+ }
+
+ return d.SetVolumeQuota(vol, vol.config["size"], nil)
+}
+
+// GetVolumeUsage returns the disk space used by the volume.
+func (d *btrfs) GetVolumeUsage(vol Volume) (int64, error) {
+ // Attempt to get the qgroup information.
+ _, usage, err := d.getQGroup(vol.MountPath())
+ if err != nil {
+ return -1, err
+ }
+
+ return usage, nil
+}
+
+// SetVolumeQuota sets the quota on the volume.
+func (d *btrfs) SetVolumeQuota(vol Volume, size string, op *operations.Operation) error {
+ volPath := vol.MountPath()
+
+ // Convert to bytes.
+ sizeBytes, err := units.ParseByteSizeString(size)
+ if err != nil {
+ return err
+ }
+
+ // Try to locate an existing quota group.
+ qgroup, _, err := d.getQGroup(volPath)
+ if err != nil && !d.state.OS.RunningInUserNS {
+ // If quotas are disabled, attempt to enable them.
+ if err == errBtrfsNoQuota {
+ path := GetPoolMountPath(d.name)
+
+ _, err = shared.RunCommand("btrfs", "quota", "enable", path)
+ if err != nil {
+ return err
+ }
+
+ // Try again.
+ qgroup, _, err = d.getQGroup(volPath)
+ }
+
+ // If there's no qgroup, attempt to create one.
+ if err == errBtrfsNoQGroup {
+ // Find the volume ID.
+ output, err := shared.RunCommand("btrfs", "subvolume", "show", volPath)
+ if err != nil {
+ return fmt.Errorf("Failed to get subvol information: %v", err)
+ }
+
+ id := ""
+ for _, line := range strings.Split(output, "\n") {
+ line = strings.TrimSpace(line)
+ if strings.HasPrefix(line, "Subvolume ID:") {
+ fields := strings.Split(line, ":")
+ id = strings.TrimSpace(fields[len(fields)-1])
+ }
+ }
+
+ if id == "" {
+ return fmt.Errorf("Failed to find subvolume id for %s", volPath)
+ }
+
+ // Create a qgroup.
+ _, err = shared.RunCommand("btrfs", "qgroup", "create", fmt.Sprintf("0/%s", id), volPath)
+ if err != nil {
+ return err
+ }
+
+ // Try to get the qgroup again.
+ qgroup, _, err = d.getQGroup(volPath)
+ }
+
+ if err != nil {
+ return err
+ }
+ }
+
+ // Modify the limit.
+ if sizeBytes > 0 {
+ // Apply the limit.
+ _, err := shared.RunCommand("btrfs", "qgroup", "limit", "-e", fmt.Sprintf("%d", sizeBytes), volPath)
+ if err != nil {
+ return err
+ }
+ } else if qgroup != "" {
+ // Remove the limit.
+ _, err := shared.RunCommand("btrfs", "qgroup", "destroy", qgroup, volPath)
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// GetVolumeDiskPath returns the location and file format of a disk volume.
+func (d *btrfs) GetVolumeDiskPath(vol Volume) (string, error) {
+ return d.vfsGetVolumeDiskPath(vol)
+}
+
+// MountVolume simulates mounting a volume. As dir driver doesn't have volumes to mount it returns
+// false indicating that there is no need to issue an unmount.
+func (d *btrfs) MountVolume(vol Volume, op *operations.Operation) (bool, error) {
+ return true, nil
+}
+
+// UnmountVolume simulates unmounting a volume. As dir driver doesn't have volumes to unmount it
+// returns false indicating the volume was already unmounted.
+func (d *btrfs) UnmountVolume(vol Volume, op *operations.Operation) (bool, error) {
+ return false, nil
+}
+
+// RenameVolume renames a volume and its snapshots.
+func (d *btrfs) RenameVolume(vol Volume, newVolName string, op *operations.Operation) error {
+ return d.vfsRenameVolume(vol, newVolName, op)
+}
+
+// MigrateVolume sends a volume for migration.
+func (d *btrfs) MigrateVolume(vol Volume, conn io.ReadWriteCloser, volSrcArgs migration.VolumeSourceArgs, op *operations.Operation) error {
+ if vol.contentType != ContentTypeFS {
+ return fmt.Errorf("Content type not supported")
+ }
+
+ // Handle simple rsync through generic.
+ if volSrcArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC {
+ return d.vfsMigrateVolume(vol, conn, volSrcArgs, op)
+ } else if volSrcArgs.MigrationType.FSType != migration.MigrationFSType_BTRFS {
+ return fmt.Errorf("Migration type not supported")
+ }
+
+ // Handle btrfs send/receive migration.
+ if volSrcArgs.FinalSync {
+ // This is not needed if the migration is performed using btrfs send/receive.
+ return nil
+ }
+
+ // Transfer the snapshots first.
+ for i, snapName := range volSrcArgs.Snapshots {
+ snapshot, _ := vol.NewSnapshot(snapName)
+
+ // Locate the parent snapshot.
+ parentSnapshotPath := ""
+ if i > 0 {
+ parentSnapshotPath = GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, volSrcArgs.Snapshots[i-1]))
+ }
+
+ // Setup progress tracking.
+ var wrapper *ioprogress.ProgressTracker
+ if volSrcArgs.TrackProgress {
+ wrapper = migration.ProgressTracker(op, "fs_progress", snapshot.name)
+ }
+
+ // Send snapshot to recipient (ensure local snapshot volume is mounted if needed).
+ err := d.sendSubvolume(snapshot.MountPath(), parentSnapshotPath, conn, wrapper)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Get instances directory (e.g. /var/lib/lxd/storage-pools/btrfs/containers).
+ instancesPath := GetVolumeMountPath(d.name, vol.volType, "")
+
+ // Create a temporary directory which will act as the parent directory of the read-only snapshot.
+ tmpVolumesMountPoint, err := ioutil.TempDir(instancesPath, vol.name)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpVolumesMountPoint)
+
+ err = os.Chmod(tmpVolumesMountPoint, 0100)
+ if err != nil {
+ return err
+ }
+
+ // Make read-only snapshot of the subvolume as writable subvolumes cannot be sent.
+ migrationSendSnapshot := filepath.Join(tmpVolumesMountPoint, ".migration-send")
+ err = d.snapshotSubvolume(vol.MountPath(), migrationSendSnapshot, true, false)
+ if err != nil {
+ return err
+ }
+ defer d.deleteSubvolume(migrationSendSnapshot, true)
+
+ // Setup progress tracking.
+ var wrapper *ioprogress.ProgressTracker
+ if volSrcArgs.TrackProgress {
+ wrapper = migration.ProgressTracker(op, "fs_progress", vol.name)
+ }
+
+ // Compare to latest snapshot.
+ btrfsParent := ""
+ if len(volSrcArgs.Snapshots) > 0 {
+ btrfsParent = GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, volSrcArgs.Snapshots[len(volSrcArgs.Snapshots)-1]))
+ }
+
+ // Send the volume itself.
+ err = d.sendSubvolume(migrationSendSnapshot, btrfsParent, conn, wrapper)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// BackupVolume copies a volume (and optionally its snapshots) to a specified target path.
+// This driver does not support optimized backups.
+func (d *btrfs) BackupVolume(vol Volume, targetPath string, optimized bool, snapshots bool, op *operations.Operation) error {
+ // Handle the non-optimized tarballs through the generic packer.
+ if !optimized {
+ return d.vfsBackupVolume(vol, targetPath, snapshots, op)
+ }
+
+ // Handle the optimized tarballs.
+ sendToFile := func(path string, parent string, file string) error {
+ // Prepare btrfs send arguments.
+ args := []string{"send"}
+ if parent != "" {
+ args = append(args, "-p", parent)
+ }
+ args = append(args, path)
+
+ // Create the file.
+ fd, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0644)
+ if err != nil {
+ return err
+ }
+ defer fd.Close()
+
+ // Write the subvolume to the file.
+ err = shared.RunCommandWithFds(nil, fd, "btrfs", args...)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ }
+
+ // Handle snapshots.
+ finalParent := ""
+ if snapshots {
+ snapshotsPath := fmt.Sprintf("%s/snapshots", targetPath)
+
+ // Retrieve the snapshots.
+ volSnapshots, err := d.VolumeSnapshots(vol, op)
+ if err != nil {
+ return err
+ }
+
+ // Create the snapshot path.
+ if len(volSnapshots) > 0 {
+ err = os.MkdirAll(snapshotsPath, 0711)
+ if err != nil {
+ return err
+ }
+ }
+
+ for i, snap := range volSnapshots {
+ fullSnapshotName := GetSnapshotVolumeName(vol.name, snap)
+
+ // Figure out parent and current subvolumes.
+ parent := ""
+ if i > 0 {
+ parent = GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, volSnapshots[i-1]))
+ }
+
+ cur := GetVolumeMountPath(d.name, vol.volType, fullSnapshotName)
+
+ // Make a binary btrfs backup.
+ target := fmt.Sprintf("%s/%s.bin", snapshotsPath, snap)
+
+ err := sendToFile(cur, parent, target)
+ if err != nil {
+ return err
+ }
+
+ finalParent = cur
+ }
+ }
+
+ // Make a temporary copy of the container.
+ sourceVolume := vol.MountPath()
+ containersPath := GetVolumeMountPath(d.name, vol.volType, "")
+
+ tmpContainerMntPoint, err := ioutil.TempDir(containersPath, vol.name)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmpContainerMntPoint)
+
+ err = os.Chmod(tmpContainerMntPoint, 0100)
+ if err != nil {
+ return err
+ }
+
+ // Create the read-only snapshot.
+ targetVolume := fmt.Sprintf("%s/.backup", tmpContainerMntPoint)
+ err = d.snapshotSubvolume(sourceVolume, targetVolume, true, true)
+ if err != nil {
+ return err
+ }
+ defer d.deleteSubvolume(targetVolume, true)
+
+ // Dump the container to a file.
+ fsDump := fmt.Sprintf("%s/container.bin", targetPath)
+ err = sendToFile(targetVolume, finalParent, fsDump)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CreateVolumeSnapshot creates a snapshot of a volume.
+func (d *btrfs) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error {
+ parentName, _, _ := shared.InstanceGetParentAndSnapshotName(snapVol.name)
+ srcPath := GetVolumeMountPath(d.name, snapVol.volType, parentName)
+ snapPath := snapVol.MountPath()
+
+ // Create the parent directory.
+ err := createParentSnapshotDirIfMissing(d.name, snapVol.volType, parentName)
+ if err != nil {
+ return err
+ }
+
+ return d.snapshotSubvolume(srcPath, snapPath, true, true)
+}
+
+// DeleteVolumeSnapshot removes a snapshot from the storage device. The volName and snapshotName
+// must be bare names and should not be in the format "volume/snapshot".
+func (d *btrfs) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error {
+ snapPath := snapVol.MountPath()
+
+ // Delete the snapshot.
+ err := d.deleteSubvolume(snapPath, true)
+ if err != nil {
+ return err
+ }
+
+ // Remove the parent snapshot directory if this is the last snapshot being removed.
+ parentName, _, _ := shared.InstanceGetParentAndSnapshotName(snapVol.name)
+ err = deleteParentSnapshotDirIfEmpty(d.name, snapVol.volType, parentName)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// MountVolumeSnapshot sets up a read-only mount on top of the snapshot to avoid accidental modifications.
+func (d *btrfs) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) {
+ snapPath := snapVol.MountPath()
+ return mountReadOnly(snapPath, snapPath)
+}
+
+// UnmountVolumeSnapshot removes the read-only mount placed on top of a snapshot.
+func (d *btrfs) UnmountVolumeSnapshot(snapVol Volume, op *operations.Operation) (bool, error) {
+ snapPath := snapVol.MountPath()
+ return forceUnmount(snapPath)
+}
+
+// VolumeSnapshots returns a list of snapshots for the volume.
+func (d *btrfs) VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) {
+ return d.vfsVolumeSnapshots(vol, op)
+}
+
+// RestoreVolume restores a volume from a snapshot.
+func (d *btrfs) RestoreVolume(vol Volume, snapshotName string, op *operations.Operation) error {
+ // Create a backup so we can revert.
+ backupSubvolume := fmt.Sprintf("%s.tmp", vol.MountPath())
+ err := os.Rename(vol.MountPath(), backupSubvolume)
+ if err != nil {
+ return err
+ }
+
+ // Setup revert logic.
+ undoSnapshot := true
+ defer func() {
+ if undoSnapshot {
+ os.Rename(vol.MountPath(), backupSubvolume)
+ }
+ }()
+
+ // Restore the snapshot.
+ source := GetVolumeMountPath(d.name, vol.volType, GetSnapshotVolumeName(vol.name, snapshotName))
+ err = d.snapshotSubvolume(source, vol.MountPath(), false, true)
+ if err != nil {
+ return err
+ }
+
+ undoSnapshot = false
+
+ // Remove the backup subvolume.
+ return d.deleteSubvolume(backupSubvolume, true)
+}
+
+// RenameVolumeSnapshot renames a volume snapshot.
+func (d *btrfs) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error {
+ return d.vfsRenameVolumeSnapshot(snapVol, newSnapshotName, op)
+}
diff --git a/lxd/storage/drivers/load.go b/lxd/storage/drivers/load.go
index 501b579f34..c9374157af 100644
--- a/lxd/storage/drivers/load.go
+++ b/lxd/storage/drivers/load.go
@@ -8,6 +8,7 @@ import (
var drivers = map[string]func() driver{
"dir": func() driver { return &dir{} },
"cephfs": func() driver { return &cephfs{} },
+ "btrfs": func() driver { return &btrfs{} },
}
// Load returns a Driver for an existing low-level storage pool.
From 626c2fcf59df04f11cee447325a5ff43e9010faa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgraber at ubuntu.com>
Date: Thu, 12 Dec 2019 09:38:40 -0500
Subject: [PATCH 6/6] tests: Update exclusion for btrfs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Stéphane Graber <stgraber at ubuntu.com>
---
test/includes/storage.sh | 15 +++++++++++++++
test/suites/incremental_copy.sh | 3 +++
test/suites/storage_local_volume_handling.sh | 15 ---------------
3 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/test/includes/storage.sh b/test/includes/storage.sh
index 11d78e26bb..c41901f01a 100644
--- a/test/includes/storage.sh
+++ b/test/includes/storage.sh
@@ -128,3 +128,18 @@ umount_loops() {
done < "${test_dir}/loops"
fi
}
+
+storage_compatible() {
+ if [ "${1}" = "cephfs" ] || [ "${1}" = "dir" ] || [ "${1}" = "btrfs" ]; then
+ if [ "${2}" = "cephfs" ] || [ "${2}" = "dir" ] || [ "${2}" = "btrfs" ]; then
+ true
+ return
+ else
+ false
+ return
+ fi
+ fi
+
+ true
+ return
+}
diff --git a/test/suites/incremental_copy.sh b/test/suites/incremental_copy.sh
index 2340d65425..3732abe3a1 100644
--- a/test/suites/incremental_copy.sh
+++ b/test/suites/incremental_copy.sh
@@ -10,6 +10,9 @@ test_incremental_copy() {
# cross-pool copy
if [ "${lxd_backend}" != 'dir' ]; then
+ # FIXME: Skip copies across old and new backends for now
+ storage_compatible "dir" "${lxd_backend}" || return
+
# shellcheck disable=2039
local source_pool
source_pool="lxdtest-$(basename "${LXD_DIR}")-dir-pool"
diff --git a/test/suites/storage_local_volume_handling.sh b/test/suites/storage_local_volume_handling.sh
index a4745b9a9e..3780900a30 100644
--- a/test/suites/storage_local_volume_handling.sh
+++ b/test/suites/storage_local_volume_handling.sh
@@ -1,18 +1,3 @@
-storage_compatible() {
- if [ "${1}" = "cephfs" ] || [ "${1}" = "dir" ]; then
- if [ "${2}" = "cephfs" ] || [ "${2}" = "dir" ]; then
- true
- return
- else
- false
- return
- fi
- fi
-
- true
- return
-}
-
test_storage_local_volume_handling() {
ensure_import_testimage
More information about the lxc-devel
mailing list